From 323dcb58ddcb19893519941e7c0f63a3bb15b8b5 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 05:28:13 +0000 Subject: [PATCH 01/87] Pipeline changes to CI --- STREAMING.md | 52 --------------- blobfuse2-ci.yaml | 133 ++++++++++++++++++++++++++++++-------- config_decision_tree.png | Bin 55617 -> 0 bytes setup/advancedConfig.yaml | 2 +- 4 files changed, 106 insertions(+), 81 deletions(-) delete mode 100644 STREAMING.md delete mode 100644 config_decision_tree.png diff --git a/STREAMING.md b/STREAMING.md deleted file mode 100644 index 99882f130..000000000 --- a/STREAMING.md +++ /dev/null @@ -1,52 +0,0 @@ -# Blobfuse2 Stream (Preview) - -_**If you're experiencing slow performance, please ensure your config file parameters are up to date as they have changed from preview to GA, please refer to the [base config file](https://github.com/Azure/azure-storage-fuse/blob/main/setup/baseConfig.yaml).**_ - -## About - -Blobfuse2 Stream is a feature which helps support reading and writing large files that will not fit in the file cache on the local disk. It also provides performance optimization for scenarios where only small portions of a file are accessed since the file does not have to be downloaded in full before reading or writing to it. It supports the following modes - -1. **File-Handle based Caching** - - Separate file handles have separate buffers irrespective of whether or not they point to the same file - - Ideal for scenarios where multiple handles are reading from different parts of a file - - Not recommended to be used for multiple writer or single writer multiple reader scenarios - - If writing through multiple handles, the last handle closed will win and may not persist writes from previously closed handles if their data buffers overlap - - If writing on one handle, modified data will only be visible by handles opened after the writer handle closes. - -2. **File-Name based Caching** - - Separate file handles pointing to the same file share buffers - - Behaves most closely to the file cache in Blobfuse - - Ideal for scenarios where multiple handles are reading from close by parts of a file and multiple writer or single writer multiple reader - -## Enable Stream - -To enable stream, first specify stream under the components sequence between libfuse and attr_cache. Note 'stream' and 'file_cache' currently can not co-exist. - -```yaml -components: - - libfuse - - stream - - attr_cache - - azstorage -``` - -The different configuration options for stream are, -- `block-size-mb: 16`: Integer parameter that specifies the size of each block to be cached in memory (in MB). -- `max-buffers: 16`: Integer parameter that specifies the total number of buffers to be cached in memory (in MB). -- `buffer-size-mb: 16`: Integer parameter that specifies the size of each buffer to be cached in memory (in MB). -- `file-caching: true|false`: Boolean parameter to specify file name based caching. Default is false which specifies file handle based caching. - -### Sample Config - -After adding the components, add the following section to your blobfuse2 config file. The following example enables Blobfuse2 to use up to 64 * 128 MB of memory to cache data buffers with file handle based caching -```yaml -stream: - block-size-mb: 64 - max-buffers: 128 - buffer-size-mb: 64 - file-caching: false -``` - -### Disable Caching - -To disable caching and stream straight from Azure Storage, set all stream buffer configuration options to 0. diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 2298d0df1..175b6667e 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -14,58 +14,135 @@ jobs: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' + poolName: 'blobfuse-ubuntu-pool' containerName: 'test-cnt-ubn-20' fuselib: 'libfuse-dev' tags: 'fuse2' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) Ubuntu-22: AgentName: 'blobfuse-ubuntu22' + poolName: 'blobfuse-ubuntu-pool' containerName: 'test-cnt-ubn-22' fuselib: 'libfuse3-dev' tags: 'fuse3' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - steps: - # ---------------------------------------------------------------- - - template: 'azure-pipeline-templates/blobfuse2-ci-template.yml' - parameters: - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - - - - job: test_ARM64 - displayName: CI on - strategy: - matrix: Ubuntu-22-ARM64: AgentName: 'blobfuse-ubn22-arm64' + poolName: 'blobfuse-ubn-arm64-pool' containerName: 'test-cnt-ubn-22' fuselib: 'libfuse3-dev' tags: 'fuse3' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) pool: - name: "blobfuse-ubn-arm64-pool" + name: $(poolName) demands: - ImageOverride -equals $(AgentName) - + variables: - group: NightlyBlobFuse + - name: MOUNT_DIR + value: '$(Pipeline.Workspace)/blob_mnt' + - name: TEMP_DIR + value: '$(Pipeline.Workspace)/blobfuse2_tmp' + - name: BLOBFUSE2_CFG + value: '$(Pipeline.Workspace)/blobfuse2.yaml' + - name: BLOBFUSE2_SAS_CFG + value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml' + - name: BLOBFUSE2_AZCLI_CFG + value: '$(Pipeline.Workspace)/blobfuse2_azcli_config.yaml' + - name: BLOBFUSE2_ADLS_CFG + value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + - name: BLOBFUSE2_GTEST_CFG + value: '$(Pipeline.Workspace)/connection.yaml' + - name: BLOBFUSE2_AZURITE_CFG + value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml' + - name: BLOBFUSE2_STRESS_DIR + value: '$(Pipeline.Workspace)/blobfuse2_stress' + - name: GOPATH + value: '$(Pipeline.Workspace)/go' + - name: ROOT_DIR + value: '$(System.DefaultWorkingDirectory)' + - name: WORK_DIR + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' steps: - # ---------------------------------------------------------------- - - template: 'azure-pipeline-templates/blobfuse2-ci-template.yml' + # ------------------------------------------------------- + # Pull and build the code + - template: 'azure-pipeline-templates/build.yml' parameters: + working_directory: $(WORK_DIR) + root_dir: $(Pipeline.Workspace) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) + gopath: $(GOPATH) container: $(containerName) tags: $(tags) - fuselib: $(fuselib) - skip_fmt: false + fuselib: $(fuselib) + skip_azcli: "false" + skip_ut: "false" + + # Code lint checks (Static-analysis) + - script: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin + $(go env GOPATH)/bin/golangci-lint --version + $(go env GOPATH)/bin/golangci-lint run --tests=false --build-tags ${{ parameters.tags }} --exclude-dirs test,common/stats_collector,common/stats_monitor --max-issues-per-linter=0 --exclude-files component/libfuse/libfuse2_handler_test_wrapper.go,component/libfuse/libfuse_handler_test_wrapper.go > lint.log + result=$(cat lint.log | wc -l) + if [ $result -ne 0 ]; then + echo "-----------------------------------" + echo "Below issues are found in SA" + cat lint.log + echo "-----------------------------------" + exit 1 + else + echo "-----------------------------------" + echo "No issues are found in SA" + echo "-----------------------------------" + fi + displayName: 'Static Analysis (Lint)' + condition: always() + workingDirectory: $(WORK_DIR) + + # Copyright checks + - script: | + result=$(grep -L -r --include \*.go "`date +%Y` Microsoft Corporation" ./ | wc -l) + if [ $result -ne 0 ]; then + exit 1 + else + echo "Copyright statements are up to date" + fi + displayName: 'Copyright check' + condition: always() + failOnStderr: true + workingDirectory: $(WORK_DIR) + + # Go code formatting checks + - script: | + gofmt -s -l -d . | tee >&2 + displayName: 'Go Format Check' + condition: eq(${{ parameters.skip_fmt }}, true) + failOnStderr: true + workingDirectory: $(WORK_DIR) + + # Notices files check + - script: | + ./notices_fix.sh + result=$(git diff NOTICE | wc -l) + if [ $result -ne 0 ]; then + echo "Notices needs a fix. Run ./notices_fix.sh and commit NOTICE file." + exit 1 + else + echo "Notices are up to date." + fi + displayName: 'Notice file check' + condition: always() + failOnStderr: true + workingDirectory: $(WORK_DIR) + + - task: ComponentGovernanceComponentDetection@0 + inputs: + scanType: 'Register' + verbosity: 'Verbose' + alertWarningLevel: 'High' + displayName: "Component governance" + condition: always() diff --git a/config_decision_tree.png b/config_decision_tree.png deleted file mode 100644 index 82a33c03e868576adc9f990dd55832cedd8d5bbb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 55617 zcmeFZ2UJtp+cz2=XRv}ZI-pWwK_Y?zf*>`csECv(h@g~6mo6;<0)&xK1SzATBgvo? z0j2jqKtYfiC3FH2LVy4XJp@9Mdjih;|G#(L^{u<^{nq!s-@P+yxrUs5&U2pq^yjzt z*^#%5^$zSkwig5f9nimV)dU3Ej|G8#`fc|Qz?Jf>lv&_^$L(*J7=b`RG9XacA0W^M za4BpW1oBq|fo2|nKpM#)kl3SFHMg~Zf#O?+cdqgId=_i%Q%hSlzJ79QnzOkX`!ZSM z@=Ze%%h2#hPv0Q&2<0oOmpQ+XUr=Ig@0y;G6&;sw+XDW`KiJ&b@rJRvp0Rn%i^S(~ z@n)9xx$lZ1qF?A4-POKol=3PA5%haNNO&49)6vyi^NNAFwL?Ksd1*y;L2-FOaoO7X zdi{qcJ>$ESZ{x4C^4z@x3{CDkx%s5w-h@ZQxOyS1>|CxHnw6GUTiUu<*f_Rzbmit3 z@w>v!0p@qu8R%UFJ^NAe2JmCgqZ{{pL7*dPf`2>o{lnvdi$Z?-M%RR<_DcP%AoESM z^EW^eApNVC?*t6Zai6CyYay~-qE5>Gc^HXE$oI(Mm&v6M;%8C{3YfXflx1kq36QJU z&wF3Imag4}UBp{}gYsax>-q>;%jQ-wjo~mizWXc89LB z5b(QUYQoHWzhqr|9emOzzKOFJpgy0GG5z~m+Z)$p&KZFIt4G|94 zIV-t#V2z!EQSi&V8K>CRgHK}xzZxPk9jAAP_D8h{Xn%9;X+N~e77WxzZ>+$zbp(GZ zywTyB$B6)cHMGBl@q4KPGhVnbLqh)-+*<3rfy<8c3l=c8DKc!7GWf`W9tj}$myH?*$J{s5sL!V zKkHATqwV6ndvXqybQfN_8OlBqK$C&EYMh+u zMZDJPr{|_ek%J*#!n%>;dokhq;AVk!9Jhhz@KMN}=2JLJpYyP%kDF{R&X3yei?XH` z%I3OXd$IAui7ZXS5??Q=cFbsfPPu5K73ugU z2c#^F{2`|2PD;W?&B@ko6-IF2_{__`5Qn!W-M)b)9I-ZQRng-S%dOTGb}n<()~wQW z*=~2&Ek&PPrLI7si3I_d5+F=%Az#&6>WXC$4>d#VtT_)WA??dC7$+G7UI>P}@kt_3 zDNSU0i1IcAN;S&dHL+B3JfyD9c(l~nJw>Oz#4{-HC1=g54Kw7&`gju0K$6E z?wP}RS4Y0)1-2Ms&iImi4rsM!n|RU>zv!7*f{aC$qqk*-qX!snVB>7G1fK zH-8(0{n34~GDj zTM40W*WnLXSSQvuZ7ItRp3P@1MrN1w{~phJcHMce?|E44r&}?1c(c5;tMp zC%3aJJoUS~{5&oM*SgQ_p|vgy7R;6oS@DVGR9QGa8=Qw7BWkuC5(uKv$(Ucre4hYa ztnb@^T;v`13JpzP+64>^M2i^lNo^za{9__XpFeG}T?<@__&D)r-xKHO? zrX+rb&QZTkGVwN$NkO2r#bP=d+e^QFY0IE8a{nKNCFN|skd=QeWA$#<}2});l##`KC7PU!>J5 z80(jACibHE6+ZoQ9MWv(!a4t?;O$U$;f+?c6{@=mQ)TF%D{n$d4|R97tXN@6Uh{yr zjwm>ew_)nI)$LshY2!(l4`*P2WWo*v=4sIly*pT+vZMzSa>5}=uxumi8stQRY&Z1- zb|oR@WBe-NLPUE01Je)@R98F~JIX!{M zL33C+*izSFm(emF@8jsLKa(rEhT5v^Pd{<-Iy-aWW)$*moltw1uJj}$jD#>7b4B3g}B~4DUEOoEFtOD1h zElK0g!OpJvz_v*jI5Wv+uW%Iu&iGBAl}rL0%TC%3yCA`nkfaDD7VC2>+fCI5HO^be z(o)|zC9mGU6+^4$Bk5>YL$^4bbS}OrQ?2(FYND<#Wg&e@8j1UC`JFdyB`(pxKI+2z zXm9fjX_joX%h=5XOBIi_149dn5%G1=>GU z;HKLm_zQ%$_F|z(5wU{$p`n>JNVu@-X`beaqh-7S3b(Kl`Ev&r?p~WO&~({2=_mcn zI>jlYG8d=`BZvGLN{ZELP{aK!ppgZAD;JIsSa28wC={O^ z7`_Ms-FIZi4+Ib{ELzwT@RGD?=Bsb7d~o-_b ztt|9p2jCa402_ALJV)YR^aDcnw>^BPh(+{1$sa+rUC@E>kL0^YKw%e$_#XhXJ1$=zMuY7`X-DV_{)E5&HkU{vYB0Uzy%tV!{~z)b&C`NF9T^xMSd% zxtCf=StuK;ZmBA85%Oxg;=QE46GpSa-R23+y{9ATZ9Wq1!xyyC{YjPQ0P_l6d5Hg7 z6=Ru_&2bo7OKA~*{cn$^q+(vvg&!2` zp+yc?^$ICxEc^w4G2g;N&%un*voN8{OY(RLJ(q&6GSAfIX{+Sgqs^fld$+yt#Qm7x z7bT(=aV(XWlwfKt8}v=@P#b13XQS4mt)xBDrnQp@^#1aAorw#CB%linOG>_jb>JPp zK^pWuaaK(c;%G~JzL4{taK{MH%2GuvW)BcpXt==ZqZD z)2TV*KZJpe6kv!@v70tEB#Ss^)~F}_EZp6JyWoqm#!kE+4lQRn+b@NW92_DZu&ii0 z!aqQFiR;1uA)v?>(}sC7{NYmcv^7OEyNI*5eQ}D+)!MwUpHU#A&nOxhI_8*^Y&Qj4 z&U-2r5tr1Q1koQ#xzLGdUHCCXV-tds5lD7rNm{{>!E+OCbZRvdLz%up8#A(yjWRQN zTC!XNW`ZqgN;TAJDh?~r+|H~KnrbWoBH*_IeVRT!FJ`P>SLx-QZ0N7@fsT4Cr!yK$ z);^C~bf4ao_CLuy5MuK<&$F_+wEEdO-Y^0z#Ir&QtT8B<@ z9Oa#l2r2eA8Bo_)$0G}`kgM-kk9>~lZ>+Xf9IWcapM`C99YdK|xWb|A{tfRpld+$o zvpOFGF?OIxUrm%b_Y76j-&uuL1V3e!m4zcTW690tCz(m%l8ooJ0bX+!qr)FG#Gk(~ z2(*KH!_|#|}YGvOUYwR0SKK(CacZ+}TY(V-RSKU7^+sanzCP>BB$O z#H2zC%f%Hg2WkL8TXn8~DbqkGp}wIByUY8fwQJJS1uJBzAEFWEQIm0@*gQg1SBW&J zw0Y+b01q~&T+$RM^pB+`Px6L031g*t4FNXr9+i)YcuM3vK;#(ETaucIIY63M^33U& z7i%sRs8TGhP+Na9Wq|e5EBcwEPo0pH{U74mcvpW~BXmKhn}oYj<`pZG$*&6DA-(Xj zu%TtOE`dv{tPWw?aJdQMq`0)#9oSw+ky&P8J0^dsEw=KsM1?yPfVS)3L(SYT1L??p z;12_;DcW)@&9q1Zd7GnH1vh5ioQrL7km!u4?-4?d>UwO{rR@w>fE`<&KD-+!T$hc` z!H%JIssRx4CNmXv4$|BLOT=-Ke}dIi142{sya1Z6sq+B8ztZ0+S! z2D{MiF=8TvuTf@J@K@=|p8~?uJlQT<8{?tKaP|kTk|aT?5^kk^2n^bM=o9!7>8Wgv z$NxAC!AduxpKiP<7Ex9o>NH1^9cgBP7u@GJnsnZOwZW|E2HJs#EEHmZ)O;Xqg^cM- zA5GZ*;|*u9{-l(Pod)W$xpQ8SGdN9a(~j6mc-4C=I^z0V1WZi|QIG4Iow>;&A2yAD z0=?~y2nH>@p%b-R$sVMUQqlGgE4z_tsb{|mX0D!wNW}SBSMcVAOW_y{yzK)F=FG{J34k*AA?_G`MI=zYK7e#J3 z<|nW%BWq6pK2mXcyYRt=4aYQ6_fZFdW@%yH_hRXahgu1N!!W7&CYaIA? zad4}FS#2}0&r?kxB|cGCjQSy9=T;jf@78XvH0cQMWn#WW?kKu102vxQovS7f{pPR@ zQlQt?0L&$kAn z!itOg&kQLMi32~%7oF52Jq4_E-`AYtYh|d-r%S(DB-M>w5<7TkL#5&wB90R)vj=Fd z)qPMu`~dp;7tH?o!?h}f_gC~l52Omt>>9s=^fm{&Xi$UJxjNBjdqa*{Ud6Xq z$3q&1zPSBf^<|4P`5G0}_ta)z(-WPTbFW9J=Z^^$sjl2Vetw%72_~YMi7mNA9LOkf)L9uO~@ed{b zD_YpTfkxYh<3{PGwhqxeZEY@+a;_|90p5n*qScU%#QI%J+i$#Gb3) z$)88MpW7q(V;Fr-J%xWvV>d`aciT=V@aT1e=_88^|#{}+v2Acd=clekp5&msltD|h|)+3(hq^@=U@uHywHcjxVGc30NU=Ng;5&Ve$X|8C%PuupSz5d~F)@#>?QsfWE*q1%>Lb9g^5Kp=04esBS zGGUgn@n>gk!P%KZ-_q*nx)G1yZT6_&tG`eBtlSu<=;Egne!Ii41qUQQcmdG#F2Lrt zO+71QWd?wtelPrmfx<)Pe0ENLm!X{UP!8-@E!xlQ*P@gxQ*XY`1JUp1v5iY*#cl35 zv9+5D`R$eAe+>4B>&Z!h7XoMM?rQ7&788WK9VarIIa<&|eV!V$1i(o;usU-9t$KTV zUU_EGHoa#!At?B^DU1*l+%T29wif`DF3yBp*VcK53BsGiDY(wG0-LeeTB|ky2KCqV zI^x@w2!9hV^O?nKM%wED9EHmnl>^YOp0{g9SJ_3zGmM+sgtxg6U^CaZbv9Fk6engsRB2BaIS^^o4XLY;p8xSyGwA z)A1I)TZEVz@`-ISuIATC^D89L ze>YW)x`*IdVI4){Oi7o*3Y|Q<)3_P#Ay7loh>{0mPa4-2*cUOIYji&`Bo!V~O@CTn z<@UL`t9HKKr#4d_On6U*UIHiE?CV&5OO)vNz`q4HLEUyv@2}II(ijA96uUXOsQbCC zIaZ$X*6!%uQ1KNDixrRPU~Sa#eftt~&NVGB?jSypZ?ukk3CG^o^X#{CYI|4eT+HnL zxF0$U?a9(!&96lBN^Z-K+_UX>uAO9_MnBTUR4*P-^w4W`IrXqy49f=^o(oQDrVMts`Vcm0M8% z3-qP9v6VbdoMd0x$iiairj?Ddk?!tj$EsP|r3lw$pG~)cyo6Ry^XlP<3D59JwFuqf z7h0odvOR@vF?C9iPg-wrc%hB9$AlJoqm4F-{ZKbhmHY?f#XHFewwGrx&d=FvGHF?{ z-m^P`z%btF!f1aoKjP9RBN>pPua+y@jH#Tv;jZaO$9vf7eXa1bCUw?I#WZC>M}5lt zjbv=L5e_GmBlUbY@O5KOnJp;y3F>4O3OV_$j0(4kfhU<(vnYfd?E1Ydgl ztmcQdV5mglW}TGp$lJ|_(yCOO%lOluv&pMZbz;^8HgxDEZjSrwscN_V72Sp*V;U8Y zr42}BX%)B0&WYL@&T(eYnzJtVgBzMV1~aET=)~fYbWgJv>vj9^eT2T0g0dQK5`j~p z56=FLDVY$0vV$=gad^sCKs_?6DIKF01Bh{N>_wIzbkZp#P@|Cd;l?YGYq7H=&q6zT zY@1e%!GVxt^9_I?WMrUjBX{Q`V(c#1xT)B_8AvW2JI4yXY+5EFwRU6MRJ`_9UMuia zqf>`F=;N1>DRYdLLNz+1b0Nf2%yW%%Jsof=<#OW0r+~$0oG-mMv{pzTJ3FP~ng%;> zyqeQIAES31FabB8B@xy_tBCj~+x9-^d#a9ak_YhOK|CzH!r!l+PwW;rwR<-TJ#!s} z*Y|3M4*Fgm!a+kvGv^g5W7kT`0={_A%Nw*XoDOfTQ|%$suF_KTmsK3~#?~haFb?n- zd`41>P={brQlr(gWk=f?*RE02qF6s!X(C^_cVAzel}fD}>kfF3eKc*Qiy}VXE|oL4 zd4HWd8b~4bb2kkC*x(rWj{mUmDO?AWK>gwKoc``rkyDFbMY0B)Ua)7co|l!f`W>j- zH3gn0)LKZS>AJ2J1XIrR@m%ABx~~_Di@+u)K@6zJ+V$~;gk(RDe(rZHFOOM|` z8Vp6$=p*aWZDUg#7OA?n-IHbdU{&e}K_yfnUQ@I5d?Ay^TeVqaG0N*U-c_J-h5cV9 z%-sRm%-qJQA2Tg)vAuK>7DYndzx`+*qjhgXBj2>_=sKEvSO*9 zF1m{{FzU~#apIpEZzUhj))*3MfDWChLH#j+Qcz)D$Qi$-qp=tV$YS{sCbW8?7iw11 z#?Uq}PFqPwee+AHwjYZ@%@xSL%#WjZcEc#nUJp~*nmLCD=XU`8>^&(~3L^NkeF*8%3qn}%dt2nxfhY`wrp)_d2BvmG+cD?qw0?IYYplgZ!{TsWu~zPtGz9)Go`rAzU;(tI}$WyIML+%j&B?ArptX zY=%Ypc`~A%1dlkXRIWSvA)4|R$?f-QnU$mch1|iP^iwy?$%TZpNq@^J)^+|prVUuL zkk49KVJ@CpS7OY1yZzY{f}Uvpxche?ozCLbQkN(zYH}KJP(o_N zK;=$w!{qnEA5gEsc0*8EWX?`Y$ov!^G5PeF$biIl`5y@o``#8LvZ98t1?vm$%a0a! zSbpyTfaHCCyEhmex4E5^o;&jdzSPVPi}~KD0CGzDr-AvWEBmL_{HK5ZF8~FAT|WYR zWT^23fZo*njoyr2%2V~*6_#4ISMtMrl*Ydel`CA)TgWFnWaxr=B#eI1_=BP{G`1t? zJ1BIsL&9NlkD7W|hFs%d&EK#Tl9r`pme%Id{!wC=__^c94I)DG-%dXf zY1vt`O(gV1IW7Ola!H}zTRLjdTu4QQ(VhF8I~Jw+GXyZ-~q)eGGDZhxBJ-`MTiKac-!B)SeXw;n+6 zza{V*(fcKLECiOJ$O45%uM0N424wMx&?m7Vb}hdNs9Mj!!u-{bk^YF?(J-shEDh}U z(|KGp;C%W!TuszsWcY zya5#Qwcl!QjiJ=Wando#;aJSW#C*V+bPL-ptXL8d+y-uQU~!B<9hz!z;LP;?jQM7N zl=@h@?NG3dTWp4I!`&{==$B09_MBsIyY`05wauORe4$=F3&cGJD zU5;w(bienBF8n;xYZf&Is&hbR{&M0~&4d_1v6dLef)9>Tt)`or!> zR^|TnMv1bdN1wTV6E_dg_UOcRg08&gp3~(i3NcPg_&_DAO}0oYhy6qP-vk2FNG-LW zwkXo}*I+|=!wWbKzccw)d5YpB!tv>RK*UOhhktI!>U|QsOS}i0MuA2MJ0^SvPhNU8 z+|EC!5#O0nXbh-F@pjV`mzw*PYd?LX=<)%MgXjr#@*ADXq>$-bsu2VYK>CeWf5suD z3NJm20`b~(Qm%alJAEdS2-1FMhIQfTu~w0`ub&Y$=6DhesF*(ylPUbhhFNRPzUVc72>haP5u;ML8r`Jlh2GVyH=)+iwF;%cvjDT zZ*xc^w)`YInF_y@1r=D}#LfMOt_9n4ozUD>#z%Hty!Uw1HWpf1C-PLl=q~`HJ2uli z{P?08d)?!qQT(Ydw(mp%^WyFJ30U2UHhTRZQ)ckL9ftj}jP@yLJqfU}`9sc%(7z^Q zP}M2D(`Ty8=QGCo<^S3@#4?(1q?24 zEgU_)cnaVj_n-Oy2n^r({_v(UEL8a4*ngsn*bNaZB>iCqJo#oUp7&r&dKV23F9yoC z-1_+oPQBmZ=cNdBUYq=>{}75HW{#A!K1{J$v%WXo^uRy%m0QVfVCE>NqaVHqgsad1 zXfY9Nc>8Q#w5+=C@le4^6HTK6H6c=3>yx-hNpbAB3_oq|_UY=+;3V@WGs{s+HF*ya z#ra0%#KV%qo(KB5FQFYDn{ZsQJzLs~=3t*`9B2ZgyP-6r&P3iV z+RJy}J1H*dzd9}yI+q>yG3e6#qS5~1^4QBZKe(~bLP|C!Lt z;-^~on~SoaN}AcKl?^kCV@GG)%(Qjn-w;A06`1Ub>mea;Tf~-2WIyE`Za+pHChyE| ziEw^fXa+1?MQ?1JCR9;}kFBm|NWnYWv;Se}9|`?qL;usf%+NC5Bk-^10 z?+J^C}(v79=4g+s94(v`BMi?I}g;YuP8Yl;Qcp~qaVBd8nt$OF$w^RRsQQ{#n zZdUOSjJ8FEtHmCV`@+f$TAG?#%_KbTY|}k#6~ju+npXaom zCxP+jI!eLWx~-jeeaHvWVB{Uf`Ok+736kc)SM8Orq?DtU=?kn=8_CtJK(^R{*H4|)8^_EBl6LKMa z6u!yHxBtqV8a)8dITuEG)bnZGsPRUhaJ?CaG-wB$X5e=5y;f_tMj^)sH{jc6V46ig zZ-lWA8W^lkaI9bR(y5@?;wo7?a~FdXI&kMGRceFR9})w>qkS?=G&!C=FJ_usS>Hm_ zPj@mJ!k>o5NA%#-y9chG_8+XNpGQo!?=2Wrk#@ad>BYalmaSm%ZAke2O9@5{^1Jf)BEx5l#YA>KqPzqwAs_Qlh z?h&S*aYB{!Hfz{Qq@-$)W2#C8-YJJUQ}YIm8K#=&=$9ynR5Tr}$v?`|A><5#+xqHG zPcAtmMboTRPW~W4c)BX*uDDW-8d0qK%Q2y zo=Msm+_b#$Y0=R7OeVu{^}SYgsHu8z(eysegR$aY%s)kxr{2ZxH_Ai)=n2 zLXO+;6EFuciO`AlI(v+NPRFehb!hW>e?PXzoKjn{>CsuYVYgu5`bkrb0HCU&R0Kah17quQT*gZ^b zim5-CRTNmqETHG&Qx&|qK}E({-~;)>g%rzZMKTKN)fPTAd5`AS6PP)OtN% z+9MyBXQ%tz|NM$wbl0>UV;ij>n~?9r1!?a;V^&|ucxB^y^6APWRO;eYqvwP{%AaefEPARUo=7#6=3?9 zkyYA|kMoMqB#7F!q(+u9OD!FdTwRUL7M|8SayWNTDn)BZ_|m(vb`9gFI)gcbUyht5 z4Tcy;dYl%jq`FQ&e)8TUZzL)7p(XiCWJ|Vb%F;KApBmvUU}+We*1eE)Mx>Er6xn}_ z*JJI~c8HT*dz3dvQSs}q6L;alL*^y!y1$&?x6C$C$|4hiUG4u zjt^ZJvJojpW~10p)e!r&qQk6#sFZ+WtVw~+m2F8n!H?mFt6z^;mV$|^a}g=F%EDJD zvDmqm_Yol^Pc2QAa(fBdo_DRZ$aHB(PYIm_2x8Z<>yk)vW^p}Z1jo2DV-VQQQ9I2t zLUfn=W_5Tg(;_LQBeWcv6mi0ZkN`oU$qrMKm#76zT|92((!^5dU=!eVbxWCuL#0Df zIKR&0m2w0mba*2}gAsZG4z=8fyThv2Rnt}}zzr@tZ6;Z-HTE&43ZEC#{W=FA^DSZt zLhX0)bsWyb%JbRCk)knim}Ogi0whPRn)*`~TrJDm&79U+EblQtr=S28$C(s#bv?gh z?&)imB!v61eqR|SCapG2LMqtkRgB)E`?=ALdUGfK5c~D_bWjSM0&+n7b8*{StGaGs zNgZ*al*WF5@oZPJPYwMPZBH#A4NXbZ=BFfxr^MOj#U|sooW?{N{pzPl`xs!idi-DyyKLNurtqUvp%kDa6)0L`vUgfJ?B4 z$V)cK3$<1rb@!rV)QpvH?QK z08uc}MtX6jUL2yyaKA{1(yUb_JbbJ0JwHv%7-vzhFPfAbCX5fnSU#Cuns}7!<_WnH z4=Kt{gp8~z5VZ8Ua%S|FoE4Sd<^eVLduhb1I`+_7jS7A3y)PohJa3q2U~ij-D!`1r zeC}ytmD!d+R)h8wD4;EU=bJx1!5deJ$Gzl`*vL*LBa*jR>8w$mCH<~1aNv?<%iZpE zq;+Si7iVmgKbIDprVvVhtW=q_r#&$Xk!?+3H3dn?i2xsmh0C5FSq-{ol>`a26s2Y# z?^>Ez`Arp_^u)o6yyRCj;yf9U-dODJjepy$<`{~(wBBsakrC^4Y+he4u%vH0X#g52WGaUa{)#x78H-T!530MIsrA8p zUmK>Q-bXzXDy7|87>6x0M|q@s#6#5jv6(y^Zz*H(LA>Uu60eS+Q|-2y=t9$9CEdo9 zPk6b75?B-86u>3UoES}vt0uM7IaCtxGT!CoLv4rDu58}Il7mmvTFv$FMskH3Rv{rJ zR)M}N6CBbrAQIO>BMaj2T7y|rN1qM%fq^S6!-s@n5KcBAGhX?>0=WW=Mg1vbQS4GI z=F?JeXYjn|@!(Y%04^&n(~>P{#`M+8`Ias(zG8LZ+<+rb0`Rm}ny5b2_y>4P45qa) zz^8w;(ohc8rl1DDZ^o;b1ipHbHFgHErl5=vxx7Lc^DMfCHDM{|((=<+c>2jp*gMKy zf62lKt!>PeU|~zF)qKAuV^eO^L#J6HZ6r>U53u?Kjh@hhkvqi^o|rzIssBimr%!!}t##2|lYo&uo=di^0H`K1|-nP=|a&q(nuLQ*F$wx3ibyL!knu-=T3HD#nY}N=L*bjL6+eH~uIOTiaQBO&HJ;i$%sOooeix@w9YdP(Ym*~)%_{G4`-D|y}xT>NKmFPnUb zgm>PRjhK_&aDQts2VtE0F^H3`fbSvK!O-Bio?*(wQj_ty2nn{h(r}BVgeLWTx62%r zHT3!jI8(8fK22GnwJLc8tv9DFIc!dLQ|ba8*N0P4YNF?@Q@p%BrAC{%dN{X6)}{1} z8l}vphcq% zH3K3VsabNbH>-ZUMKs7?17Xh39vu#r zww6p*DdwGxKzz9or}H%);$`toqqCW>MN5HsdOmUg_;|^oUt-SxAusl5dEG_F8YKEA za3~q$(u#E*JzGm(10_}@22^rEHCjh=uI20tD8wG4;1P-V+3Rxr)_k=Q8MZF|&T|ch{9XW=Sr?90cVRaC)0&ZSf+`&&#Lx(aQ}Z97UHTQA$Df zysfMUQw<8&bgU8A99`CKOXKouLyzIccQv)s*p zDP(=;C!ve;7WG6gG3X(R+_@;PCBnzw7RP8Z@8a4=GdGB34yP?$CkOT?u^bcm{Ux7n*(p`wUDKGQDvrk-4j(Q9&*9 zS#{kl>^tSei_L3`Kk{^@u{Nz}?aL=xmvHs@)e^D9k`~A}cff9U()L_=t+nR}92Qt| z(Y9aDE3d3zGs`Gyu}+NEs$qZ6J{S!DwlusN%lj2Q=`3)^H&0w1u}VZpaZIIIZ?551f1 z_v7Aw7;4Cet;4UwZsAaKr`>>WW9qWYgd6#*wlgY!wg$caTqq!ls==9dsPvI>AUVv~P z{|{kE9YTY-nkeBc>;i|KemSK2)H)+ZsYJtwGb!D)1L?In!g^x{RK{5^oZHmxD}@` z>1Y}Sll^Wc>q8jsEwgCj{$_XrM9)6TJ~4}+B})kHx4_0z#UA_mA^=1@(eP8R?U;7ij7PW zENfs@z42_S4bwykcS`~j(Zj?0Mj7r!;-h0{6WT`KyIDaZ>)IFm1AUIbHZ*hsE>I`$ zewL}N(Q*m#-PC`%uEz{k)Ol+PuWu4V23x)%==4xY))iIR( zKY#qyfTs80FOK~K&L<=pnv7Gh`=+B(+DCaL2v7vn_d^6xZeeCAQ@Qg^-2Z?mt@YN^ z{HxK#swFy=Dtd&g7!me()}gMB22m3x?fB7Knu+{D{Jmv z_(vF)i+11p^I@isz5n6C9+r=bD|78}_AR_T0nX*h> z_(VPWmpKl11Z=n1v0)@dIECI!RBK+-2XOc;WBz!4*uS!4dAOgN>VuI}f+a%mwcwl2 z^Xjmpy}J4)3`Wo;sc7{cd}dkF>8Xg1hq>NJGi-cLrC5YeZz~IspdpeWv6xWvJKQ-@Uovhop|Kd~0cR{A}XQ@P0mG_+NPBHVgW>rR{XRF_R>#H5B{`AB<3y42Ie zhkU)K=54mFHk}-bsTUg1j=Tk!%FG3p^ePikm6SG*(h<_Wz2=J*p~${B*6Ci6uu+mJ zvx|9FYpB%WEJcj*rz^T(lbC&^reK!Dnn&;(P1UkZ+O(^Gd9w|2uC*OPDxNw@WXpME zDe0)S*mOIzi&=hQRiuHr2l8i25NM;PC-T6d7tC%~$Mqr7Zp}AqBG;GqxS6W(IQ82~ zBCie)HYac(L@BZ(@_pUBo@-Q{$Ls834`jeaSr4~iyswS#uv46;L>pK;7~v8;;bjd} zXjA=qh@sO7HTL7W!-GiAE^`r5t-a-dlHEhix{)WVo7KudXXl(l&MwW-K{Bnv--x5R zPd@T&-ZkE-?L;CzIp7yNp7@M+TEKq+R}byl#>lEHu>)MY)c{Ejd@sO$xLXC<Gcvgq7jcG)6 zt@uGjm1_Hq7|Dl{kWbzS#N3s;e&KE!pS3Pe1$Gd3QB6s;m_67iE;}`?`y-7(4`e?t^vMOEum1-8Pp@LU1Dl0QYj0wz20LK1)_!qNH4zpoQk+~Jy2i^~ z;|4r!rB7sTvL-TziPGHl!a&Z#qH+5;=MunMVmu`04P>?Y?jo&q^(%(Nd^g3?ZyB*k zl=``<_HOb?Z+<-W+OOubEvrnP@8XMmqg&ewMbTOUq-1&a~0c{@y7_3lL_l>OIXpJi!xz ze%E{^XYQly$;|0g4?A>OOo6I3llcr^nj_Jlv;1@WXU}MZEsuG#j87T)d3AQl!CkuI zD2>{xd=DMWLZ6;wl}tZt_>Ho1eroNKMAwI%rhJ2~?9n4; zp{=7u?RSGAs?lv6`X2{5w+hnn%OUeDnq#>avsp0j~u1Efet~-f*_;CHpZ}=XloAbkGjzf15#-4?_ z4@s^})2g0fFt9~{SSW%TK_#wIePQQmkt0WY)yn1IjOCJ&O^c3qwgQ)TH1$NCMpDd~ z`d8~@H@Xk&B{4thgnxm$K4Mq9k(_awZ{kt^9-j1rj=^UtwC1=bC`t>Ega`pbO$Y%( zNc-Km_I;l3yx(`;d1rnz`-d}f-?^`Hu5+EMoX_W+L5TtEb`A|8C|!7`57yIvd5y6` zfie#HMqw3nKNbiosf+DQvgX-gJKCPZ#pDzAI1^kPf_!n97FZSmD zYA}!u3xF^IM)*HNnA(0*IWtvQ1O$1$0lnypj5Gj{T!upB$k4DTSV|5$>@n^fu<0q@ zdMH|DPzr!>S@@K`+50Z*??3}nZ?^H{_>SVP(!=jzY}ZyXf}dm5)b+kSkYliH`cPJm z+o;>7+VCFK_)_`8Z(<*VMdQjOjCUY_I=h2&_J`c&JO1D*4M1f9NZk9f=JWH}Z+<2F z9XrzdW^SKhVubiGZfM3Ja&8qp_f>(PV@vP0TL7Y!7`|A{bmO%?eYe?ZK2kc><@?`0 zf+2tY6m>e>lMeMUPZ2LXbK&hK=I@7)AsfqLZcG%6dc3mNya3;OC_44^Z-{y2q5Ylq zuToKn-ss$OqY`S*bdLNCpzOZe`kX2Nn*Pi^sd*Js`<(IQ#wtv9W50H*4-x%{gef5XWC|CtN5qBF2R?(K?50RYptmVOei$&RyKW5Yp;z&ZP$=c;V}62bo6 zNkxrA60&P6V;p~%1`c7rD-GY{O}^go%&$TvXkynhm(2!Wg63~{d*+t{iLNeOZTzPE zjNoJWy2a5>6PnNLzWFBp^~GPkty2wfq%j~SvG>I9;R$!_eM7(zIWE9G*I)Iw4H$sY zd(KT z8{rMb&O4{2@L$D#(?*{p5aAyF^*??+xu`m-mGMfGE>khDDN9t-2{6B37FUY4~JJfyov5#)gOLxk1Xp24~4KJ7tqE@8P z8j9PfNa|H#;l&tD?WEqfM{lhWxI{GK2Ve!^jr-Lrz~xVL$LSju5jn34R6S zroaQYEMG1Y(!!fznQN%49I|z$W=iITQ`c zxjomI>h@fvV#GRrQEdcr1bAgN)ot~jrSavf+7zHhhdhYGdrHMu!uV;pSyJrglfPf} z{XBqCVpBy+wahC7JDrx*_3s`}eN>oS6MFEK?Hb7WY}~4oa*`mCdw7RMQ_>)`0wAu$ zE}dc!?}hoVt4mv1NCxO15c|);{JDOAzE_Xu>ke?b*Y2}=p7Eg_qshs0)-P6r`#0!qA5wo$%i_($bmLVwK7>Q13gYpjoeLY3AKCO@@fPf z#dy53zU~7w{{2XN+JMXjch_16D4TI-{hz@@D*AgHkrHZzM5mS7vzKy?0w}kXl zs6o)h6YSQ+Q4T4yfDXwD75@1jC@pIbrS`?>bT0-`pyQ-3H-Q%X&1M=Qe({8lJ{ZrikbPAS8y?Rk? zh??1x9NVmSE^)T)6@bwk!cxV#A+tu7%it=8DM}D@@sS51=T_EFjC&&|tz#vl0T0(Vr$5v$6jv+e6qi|8l!~Liy5(YI46z zC#B^&0E5j-3efUXl8N*<#BvY76Bz{&J&QD{@kXg&dQb65fz zFyxhAl)Ktu^|*qN&@KSidX3CJ=NVR5ClE|b{S+2{nb|67Gg`hBpK$~-lk<>lZ=b*o z91U9fOINEpX8bgC0S{&9cxealPV(M7c0I;D3>iB6Ux(hSsU26keu18QqZWRxPzsPNsw2BN|LUn8(;Kt*VK(EH+>?gTrdzNt}O|0Sz=(^2l2G!UR{dUV?a49zog@$w zg9d-(nSDtQyRl(hoz`K`nDc~nAr&CvxrYstK*5t4m1UsTgLo7u4Q%LBh2`7Fx<6Hb z=>OO5ws|(C>B0TeWu?IKdXpVY;fl^!6tcpIVprGXrd$a|lY@e+%2>a*j@^t+0%bq^ zZ;i=4L34s?cR!IPNQT|yy#*XzG3N9~ki-(*8krw}ai|{})Y$|Tav043Sk5V34l;Hw zOlaAf*m=zN~b905dlf+DL(El*G zd}hq0x&=9G-dtqEBj39ZcqO_NBu*8?0m}pPSscqg$RdvT|RAp~f3V-oiBDWB;p8^u@&YE<1;4Z0jJa z5vJ#x=U;*)aJ+f!WI6?a`;t1$X~kxYd5K$9&A$X=hlKFg;m07sv0=0vEzwY{M6~E4 zy5G<>rL!s0@W0NS9@k9bq>2u8`s9Fo?`SJWm^hjUv%WpOh<7Ec5qpOc#q z!i1Sp`##hsxu!#k;}4lagH}Q`te-6FN0A}#nTl%=`>FQS)kXT_ zdNlq5@CwIc3TGv&gJ)f$;k9IzzuIeEEKX%~va~saAX}hhJ~K0XTMIymF&t6~-T|RQ z|Is;#zGh(^fntAQw&O%8QOS9Im|9~URB47nwFxV0>CnIBxy-`_X~efh5~CMXjr3P+ zey*LCk7*ffNS%67b4H~~%3oA!yAC5KuY5C9?a=BH|IrS>p6SnzEZn}zaplcCZ9@k- z&v4HqBebnxIPS{cLOy&zSik4cTML)gR@>QfU%{3k%Jv%NCFza24d%6a}jz;tb9fF;#Vr z7j-!F)krM+vV3>v4)9L13Z-k1c15QEN^I&c_0p1n)bq?g%xPU)`A8P`xlally^qKK zlPp}eb9Ac_rkxkgPj{InL7Ups)kNc2$b&ja!gB5qU+$N-**j#svenD=#~u&GtZvVG z^62sJ8MJc6@pbyud&q-?a)`OJhDMP|wl(eHiFBv*T%Vt8Yin2?%msUl*TYIAk9kS&LrcEz};%U@CX+8aUvXRXIrF&zTm<64EB7BVQVDgznG`v4W1GN0EDRa{L`YPaM-j@bG){-B`-}2z zv&zS;^~MB1hSYmYPHL5V(lXKlPY#*QVJ3qD1l7cudCut4-VzXOX%UWLbNDBMfW5>Q z5?eq0aEVhwQGcuwm9@j6=#OXgO^Z=Fi-*fZEzl@LpTi2!g{ASzf{gbh&)f%jzeuGbr zgN27K+zGSF78jAT_i?P^f3O|i&h33InD!q_rfq5ep|}>*ajWd$_^b72d3`Z#N1Sa+ zt|p|I?Wj$zOb)yDG|UhFJ;0B70Fk&;&yL`=Ic7RYWaQ#z`6ue~B8WQXG8D6vBHze% zVqTf6;8~%uWY~e~XUt-`CQ!r|ExV4834?l)hJ4qM^1V9EM=5^&7k93&f`Il6S#AN# zwmRg#HUuEpS1|8&B(;D7oAf#0vwmbzT5{=8q^>6psac~X`+&uK4Ft6~NwZNK%D9%Q z>(&bkwZleLARiTo1gh8$%F99U&y|6sToU7on3&aBqu+kLROp=QLJ#hYMs&WQEYzv3 zho13pq?&J`CSMY6y*b8w_jYsQ3><}_Lc%Y8HN20fX&#^;;WDLnCcx*KkJ6~Hw*(*t zy=PCp;T5RKk~+%61Quc^F#C=zD&MJc_87#O05LLiNwg zcBEXkF1i6icsBKNuT*un7~N6m@-w*BhEpbT{@NA#c6b7soJ31IOu(rVNWtqfPSWof>x znLZoI;sfkVSYlX1%m6*u^{qWxsw@{c7q0ZhG!OWN*OZfFU5^bqYv{`?g=QPt*MMFP zswK6ue)+`>JBr)od)cpS$2ld9Qq8Q}qgWd)bYW60_9lOK_O7p*U^tXk?+;+A7!I<8 z-p8yO+4@;cygUT)jPt|qO*i?Zw~2zwcdGaNm?V_+xsK6&5y4lk4<^MF=s;poFLyAy ze9V`P6*?07I-X4v7HW6p5*O!iPAL`Pp$N1pbN0Rw6YkAv#Sp)k;j1gS9rOKcDR5FK z=74fBM;`hwp3}LsKMi}D9kSF2pjTn$L1^jf?!$*6&-QZ6%(U$x#Zbnw)71D7j5WIw zQO8CFX9JH1ZMF`|t^;RgE-VI`Qz>SkgR+)T|DU*bGybt9(x}j#G>dofG8=1t?5xn? z6t+^Kt7SWtKJ7+0kG^Uy=P;NxOBo8+SR0JbAgBzCPvjC9=U1i1Dkn-3nXj^N;F}K*)u+cm*04 z&{qp^>P|ebezM;2>zVde!Z#1XTBj#B+-T>vGd9RfhB%Q8t}&mG28JxSZvOoo2?48` zK#3FN5C@K`wG!A`{Me`S4f?^#lX!XA^Iy)a&tVy;>@lOzn%Qw~;GgsPx>|#hxUT6V zx97j#vnVL}BTMD9qhjam*^TBh=b*y3A*C0CYTyk+D~$88M6=DJ6rpysV;6bA%9!^H zfJ-VPb(TjmRqbE_+*isphAQ+e6c*~dQp}3l`yYD^(3_l$u4R{{X3ASxql?p+&5;kY zsK3GfC|nteVc$aVr4209Q>ja1!`>jYawj6O%azcj$#$S9tf1b+j;vd8w)auJK_0F$o@g zyjg!IHBv9S_&9-3?`MM(@fy+PV-6h=nr{2>>|99m`omX43xA@XWMh0>g)=nobh~TC-%9hF}Lf!>1*TQbc^a_OZ4|uTv*`*;VT1 zeTG?wTHc;as59fSEz1TtyzZ5b?xob8ssNl*=-Fv!K3~z(hEtZcE7x5u{0O4|>igBB zq9&aM@cW)aKWwupb^I~9q0ga~$(ILpmS#5T1XQAQcAMA2D(Y?SDun9bBn-GEYDkFe|3-{u3@Mx0%+6%gK+~<&<$t%Lk*+0Aj!q_}(IyA4C3L+$Sjd>i_Tj{2%LPwp? zUh@?>8M222C8aH8S{<`JZcKY4I~8*v286wzu+nvK`@oLB8fz;=&Xvuh$u^XVqYO&p zJ}*cuoVZ;5Ct3Xoi|x}~KmFj2rv`WVaNR3b;f@-l_v+61w~sB%z}M?c=CCC?bWs3# zTR)Ds!Nz=xLGbv{`N3^EY-ob+GJTB{rn_-SGW&|KNHG?F1d{God6BnX!NI>qCwFl& zKk1=yDg><)RKH+|zL@dO|Cm?wdWeeo>hP0qF-5FoZ)o22J*3@D!V`SKyxrvTv)Q4l zEgTpp$%pvV7A3RWW-t-3l@drv^{9Ne-oUZOR`9mQ`kleSiy{LI-x~RtF zCA)F?;Zevr5maN&x9I&!S_jM=|Dam)=e^;BEV;Fc_jv`LK}GTo!QAg!6V=nyPvfq< zD%AOXvXALz;kbnxI-s{Z;60B)T3MSkYdlR|P47eK>kbtejnZHV;Mo_x&FwQnxBXk~0_3k-;y{^a3eeynHQ!_Ka7#n)V)Wcm4K_&Q zkyY4_Y?jcj$`BIDlNgHv*Gx|*mtu78F~~!MgCaHi%rKsv&y=}o`rHKF*VEPHeu44b zXWlOyO7vz|B8$yO?Ho&6O==!P-&^^UR(R(UJ^)uHU>%lwHtc-Aa74@>Kmg211qfPv zxGL;zJ{7Z14Vs+>#;mjsXH)U0Rkt3T9)~#{172Ycxg1K zQHM-^;=hRQbiS4trnjgSlQpvZPXG7S1x?y~bVy#4z(X)Neb__yv*h%X9H@FzC%omD za*N*4hWH_vAhGZ_^+7%9;n63^gfkd?X zVofWVo>&7-qod|%d5{hcJW%CLvBee2arV=BCo1AI>IH9T%YnLwA?XO@6)(;xnG8C} zHZs8tRMEU(>u~QJaZpN zLg`}qHtLo8z_gjdF~^L;L_=amcZ*T@uW`BNaOcvYd@putt?Wn zU2e88h557Z@*t|?`B+;OPbE~bt)7OLSM)F@^ohw8QAa~82k5*VxaiOIRpaLYBi5F> zkLiK04}`p%lTfUx3RS~4)ra#&=lyP3_h98QuY))0O~MS`-w_SkQaJUxlX7~TSHK=Q zfRPNkqY%&Qu-F!v5s9m{%b=n++lUwUySI6uw=m1)eaPZ( zPkKhwK!bTX&8*4g3SGabfG~m}F(3@N&2f@`F}KYpVM1=wjmdR(kRGyr6g1i4kmsG1 zDqDoPk`=a8l=XD15p2;^0rFMVK*r(=uTNT??Mrq^%qki(6Av!F=IKV}zNxCk4a^b} z=0}&LMCXQ}l_7Y!vgy5L$$EXYI^u}o^UQ=UbaIx2_3;%N!(+-EJ!B6==GHu~2ds}S zh#m}W#wheKiM3Eq8@@`4z_NZzNO4I&paDJonuCCy%_!(3?eC$}gQ~{omXRUX;GU-$ zsPR_C3*(C4dv|~``_*Izm*6QLeawvR#`Kw*{o$KXsry)%Up9-X?Nr^Y|u84ag{uBNr41x~@qFM9%87z=&uif%)w<(uVq zG2J%J0hW9eKh!5bRM=CmiR8wEr(SaWH;oYRZWDED_lvk=VZKOXc*w0^ruQqf`E~7C zo#f%QShuWx?U6ug2(;+sq~fHIybXHj+;qe;i@eP$y*C%q0p0a%;2E#{oKxqs>$ET} z^gb22KmFM^r#p@5z}cKK{}O0T@Wbv{jX~g_Syk^O&h)O}_$7P_s3qiZS&eo?#$W?T z4Mmy;Vm9>`o*icnJ2$;LhJg8pMl1t~LhV`DfZl8^%HCJWw=n#y;AEMmfO)%?v#|Ly zAMewYr}t}_z&v2p%lfy-5UB4^Puj@60k-ifcTEzVv*H18*Ti#3-51TncEriP-4yUT zy}r0T)Li0L4Vc%S2|~6MAkG(N1?*_{T8^PyfC*rez`?qf)K13ZbXnkR zUz_mIg|6YyE4COHYfIqved))WG|Yeo^F(h*zgp}3BW(~X=_snL%cWu$ipO8~D)l?g zRZ7Wr?p~yDOjx?{3m1^&HN*)@$hJGb!?RB@Fj%Y&k#S815<9n_8xnVR7!r6Lbap|w zA;ss1cO9|=0|Do*XgtZte?m5#ri5t2we7n+sWSn|a9&;D>bWJ{$&s`Lr?;?pq70Y5 zlUG2;u11L1_C`42+*<=(Av9tGrQBiuS-&Bf^MfE29T6+PNykEdz33$+n@!I;l9n_- z+Wzq>e{Y|mYrcPLfX1m;9EH{K2bdKkZT&wv1T`lKde(hI8idJ!Aiek--Z6kRld7ag zaQ-+FQ@-MX!RW%&K3s`>rsFc{mzyMwBVJIl81JCO zad$AEY)4tC^LvP`FEeFIAZ92b42c{WjzLtwf?F3yNa`3fa@Xd(Vi1b-i?e`xKyx7c zOa+KS@q`?K8XgIYFQ#zjAJx>4C1IZy)RI9;O8oFUPY8+T>i(K`VgC6npCUaVk#JYW z`~hw_w8y-HCO2gKS@FI&U1#Y)vsx3&a-99Fo}uo5K@FY$~by`4DY zyQDg4kK`qPiAErj%zXV=JG~+RSTg-!U2w$2w5A;n7l6XUe0+e=CmbN3LH|W+c1jX9 zE+eI=Z6JFJJ^5ZxGR>s#0`;R53xq$&q7Ezxm6IV%3z&Z**q5pFl+!u!SQobcpJMwJ zb}V9llR7}67s$@$r8Sd>JgRZ;S8|hM zkeTBeU89-5asF`6$e^REL99c0_~Xq%FxBzm+Dy-Msio9G ze=1^J(0Gj1FFkav^Q}UZ(>kcK2bY`Fgkq5n3G z2&@it))?B^75e12K~~TG)`g|a=)h$R6$oAw<7PTDc%nvF7XnOI3UVcn=jp-uu0Il& zLX<#w{Z z2ruE~!g)C|#Ngr#*who*S%exKub|RV5?S-88onL)uA-j<%0a|ziiVHbZkVz|wpszgx==tyAJ4-dEquJ` z{Y0I3lK75nF@OAtlmmLQtxrhv&p6G$B~+HX{pp;iCwYyELg=Bvj-64z&-PY*VrkJ0 zG9g*d@MC#8ISoUP`_Y&Y(e>-uR-HTM4lKHw0B-+JSPpiR!n|nFMigt;qsm z7_Nqm*_=DbW2aZxlC$=*RF8YSpr-%m(=<094W~1x9-Vu%`yGxTX@hO(Uz1j_m3_g{ znaiBb#X&SGc-#N@y|WIpo!`5ev3HxRZoq5d*gN@ANQ9=w#_#9yySKw0n2QG*8so&2l{Q@v7Y1+!MV?#61#~rBb#&Hx)R;Ik*8KawSsV#kN;#G>*6GO)W&}9hCo*c`zIqI#Vx^212#! zYxnu~V|*yUaq=UyhaIipA26ty;u8%Y%*Y;WchmAv%!WhZ1;2jOj1UL4qF6nZARvr` zcQK#9-%tq!FF3VQu{BD-)nwGjk3q_s1ZmIC`CuSnE!%6@pdDMfM`%Ty#`7obMduw5 z*Jp{~y4|08m^Eeu7M4E-lHU6shje5=)a7peC@bUK*Li?{U(wg}KIztSvtyZUt#PG} zs3*4ky7>gH^}R3XLt9cwF=eG{7L&P6!l)99tayh@sik@j#gxtXU>9teTY8*L@%Z_) zDP?wyIdC99h^th6bq|%J+tS^-AY2X_Lgnhn6zh++#=xo2)66bnl6T_Z`l*u$s|r)Q zTGJ4L`eVwyuCB*{GExaRZDc_3;esN-^Dm+H(F0?zdfIHG(+bOP4|fhSK*FreW!$Dc zDCDtTKIqF1TVQ;l?8;@*B+oJ(T70*-&CeY8WoW~NN#E8BSshC4 zbs31LPRxrGP_k((_QW%3sXBb?13}NT$BdaJl1o7#Hr0cZL}{SjW4#0SWp%t=RH(|A%AI%Ib-4L2pYh8J&R-};DlvxQ zbQK!p7Ec}oH7yz0V<~$pj_Q;$Jj5AQMnHrvC6i?4eB-DZiJrC2Li^OC-6mD2>S^-OQiPuPF9THwEoPLj`X?{~JKYa# zpD`O8w+lIJLx&s-!tCGDJ;Sa!>(H85f7E@(+@+iIN=hF|pMS~F^Y9ve6BAj+GtL?w z3N*3F&h4wmbC{ic@EDhcZMV?lJl4%?_N}zy#)yu>vIKJS;~&vBa392I7PYZ>H zqnQElME+zghk=2GWqAJ()$9%AZgEk+@FF_Xl|EZXXaEcVWqy=DlP9>`)r-2jrV~|7 zx3-KkyclQYtD^r{_`9*6(Ig58c&E;wc5QFoz;cxPsPXi^=C+#?G!{bcXFQmwF&nE-1{_|+> zCVy;}2*CK{|G3C`)w{t6aP4xoH)t2mbVG)u9On32ESmS4ma>mmt0M0k)p8(UY48cl zK;B-*Mu+b41>voFxyQ!;HK5AM0nFn`|FD8452OjUxXRn|^DO*) zYIz?-XuSzwtmr}?^TOjQKtK+bb;+w<1$zgAq)r+xyohfz!mIXW^)4}7G{#zdm~tqR zof*qtAwc@5hfU>U42QH9RQYGX`kbFc%anp70L<5f{llV?;jWF`^Njy1JoWT{gBG^= zG_|Yf>8`at?qAvY&1NyhheYfaZSGepEcJtiD@OsB*CUDY!#rgGQu$o2+1m^FuVYv*$5t7V5 zPBnv8w*D`~6h;S9`zrA&(oW8`Kev7~YBL07k#O%rNQ<9ShT#_p*-mho?!2L4<8}%M}T?HgK zVZL@C_eAF80|~U5Us9<^vhAC)*DQy&Z<7B@xD#;0^MLHy_*T=MK*A*;NXf?(C^tAq z9}#BCH_5)(KdR&rr3Fw-H7Yj9zF7Mupj+T!9tluS>+hraHb6;{rx$SB zF6nmNiChnJEr1%_-l9av24Buykl0?D6V#;;QO0L3eO9oqurZu;%h-%n!m zzWh7(_rLj)VkPfcerv_9?uk2{Ot(?kXEQCtWuNQm1=|3_f_ir>QMy=|5oteFbdg2UbG|1w<_RAX|T1M2`ZW}5r&(*vw zz9wYN$C?hQ*oNSquX5AHQa-fkoy*eci68q40UDOr<+u-Dql^e|wmr2Oi1_kZj;?j? zLFAue-O~VZQ%p?B09Lx0Yq0q}QBh1RMEZRtpBjL2T;(iEnN`-Oe>Q_EA@&STm0RCf(N$V}XZ<0%bf;3%73gSWRPg)`C(_ zR-!`JpcHxx-2mPpzBF-8wb_% zTxO8fe@<;!)ItLN6BGOMrk>7Z_NnzH*G45gQJL9_=W+xwOPM$%z;zT8yRclK0{pKr z{XY09#lft58py2VoUAxzw^Mt&nAn3iLPyPmj%(&4D@gGWsj+?e^vqTr#yeoL8lPwy z&2hS{2D~!Xtn$2X4r`X>19FJP`=4nBQaX8ZqyTc>wPQ@-JLMx{V#RO$_2{dQ=}=LC z53x-6@&!mtECI2iisXsv8Q4C5_ru(^ia+HN@Ak;RzZ3Ja3s*j!_e3?<_QjoDqY~9D zsa&G*DA{tC*fT$vg^WS-b*1v!{82g)c$X}6Y?F7NINUeg`ThsqE(iHu^i_*6yls9x zxfCL$-at7;0qBZr#VSpCT6+w+YW*$pbFUm?xt#1quk9lo(O%?p zt66zuIncc_QJeRjHDWd?@Ao1K`t7Ulw$Ou*cU`Bdi9Jtw>;iD_h|$f4!W+KA)*8FJ zd`XVpS~0KGm4$Qwdo6KcWb(nPMSY&~fvdLl@hYRy2(T+x8$Zy~?%kg;H}Zsu{m5M_ z_D=o7!>#u}q>8Fw#u={am%`%YQ2?GVf8raup_rKc8N|vt;0kd47MFwY3WL>rP(bhu zpe&aUWxcujNPcZd5_0obPw(B6p6y0PuSVv0UxDn03J(0uC#(c$Z?dbU1buAgUqdEW zSj(#xu~;q0wtYGV6@Zu+F*nvGYjUNQ zp~Ig2LH#4mjS=K-FXfgjnwTQb|4{pxvvXDUfM0#Cvv8c)H z5uVifAE<0=Wxi^NaQsn`moDQ%x|f?`lQ9iI83n&?_Jou+h-!keHJ7RBc2hzGe6sF+ zwJj#Y0bNY8-uKZt8x`i+kF}P%66TM@(;l-7QF)3wc44Zx$E~wcDPyVKZGsH_J@W6K zXh^Q|8N|ejOzW3k_RzcIfYTkDJkW8E0>`SJuJ_dDmGbJ@ zrI=XtNd);^ocCDKuiOeCb<5dEL4CdLB|r2zw~V8ZCir8a=B44{1W}=`QPalJg@~D{ z!6dfSMfAJVpEY(I(AawNFme77Ks%B#xe#|4GSj8u?FLCQ2CiCEk#S@nD7Suh_hobR zyWX!p`(IGZLcY*l1zKN;PL_L5%veOeH#y&;6=_ zX~?oAdcm_p{=gT?w!3OKujf@Da)PhV&rzNvJ(vJUS{I#a7v6>Iif~j^?pYsz71#4K z!uGEx@dW@ewoZ9Wn7Wg8{D&hMsXz@mPu09h+;fiu??g|=d;1Aq@8YBh=I}4fX(chA zRiWH(`f@BJ0xCJ&rdL*{h&%@2v)^Hr@EJt|2n9&#tggCnYE)6A91^zKxoUd0Gjax? z+7YWYf_J9ei|WD z1bH7|P;Y(z$yBH>`rTb!cfK%|J+jfVj2_+PdUW@?i{UC7JhzUmnqnnA@iz0bI`T$m5gPY>sa&z2Q&#+p zaR~bU4SCE%t8c_!UIhEJR59TP_AXL>1KzB?ae1owICm*4{=g$##%th>*j8gf^+76* zegAe3R1$b2A$BcWu4-P_M!ycB91D*+4-WqMv^aOm}{$`xzeZ%^cP?ydOj1L|Az%S^^raH4$;PS!vcaAtYeQ0YYC3wO~ZR?G2ylrUp9b8vxLdydsNri>RdR{ z`(%sh0@Wz(S5~Is z@H+YNll-o@cnJ$VAM=>P<%;5hl|Bb4V*W;gb!_(3GX&2Y_wuW5QUE>gbz<9=bCzl| zc267F$of+8fyA+%%aNw1!tCtI3{D0QjP1>+(y}13Xy>y^^R`Kl%4wc1v;3XEo;Lx3 z?3xZ?b(6Tf#O#aUqWmuFTX7FZrFKp5qi z+tIP%W0+$0!83yXo1autzMRK8jr>0$c!6feSyak7?0Xf z-mSz$MPIZ|6|pKY!3;@hG5WGUJYg4OL{m_KoVtNwqB!gSAiz9On5vETinBPG2yQXg zAmngH*aFMEx;!k@gIkL0A=}HNcEXpZ6=%O{29{=NWr5J59KY^fOyWLzxHt1gAa?D$ zF&ry!eH`3+7boe?j-bdlcdOWXS`AAW4^8>jNeLd&<3}sk6G|LFcByGb=*M z&2MFnw0V(}KUEuA<7;SXwV0RI0ls2p>EWQ5y=%|NQja3hEG9dHs~uh9F4yqTw0iG~ zi&rnHrd!UTV0XF|${|LUbKXP>p;2X%i%k1QHIU>Z5GN#JH+F_pA^9rkM~$NrNR&f| zT4)XY!Hj`9d?Q9i-C`)#E>P#VZEsfY;*;oH={@6w<@Pra(1GOPeXPoOVN-Umw_13Z z_?8fo;8$EZFiYm%_;iiEnjba{pKt4dI&sCW>d4Dgl~ze44M-`KoY-Spy@Wjm4v(F{ zmhwk~^!XZp&wx3KPnQN11%r5xAN)D|9jMa;p%Jd>&{t&|pq(P2KMN+T7lV`o6OS%v zMJ6(#t2F_iyo8b`Gde;_6{&^j#)YaV9CBs=pDiJVHVVr+heMWy9~kDbMz0)eCGBrY zl&5bZ#5jmsYwlM&_OrcU#TOMS5c@f|-s~V1Zw*eE6NDrn=g(ThgAN*5CduRb#>VFJ z9xPAwG=m8oFNtyNaWQ@B_mlIg=r*^Tu70Kty&4&WrS&^(l7ad>u(ZC;;$(sEyg+=> zX}#QT$OBJE^xn(|Jxvt<3s$|QL6;>Tc%W1c9ZRIxk7GErlcrE)bw0LO#2tmwn&*=_ zHu5e5v%eJ^*sC{dZ)eDCysX}BYjs<`k2r_lY}cDN^>I3w#fbBv;j=OXr8zd z^|%Z@IE#{6CN?ErfcfM$0L^XWPr7V91|gujL%Q0)oe%d3J%ENL#K+s%v0EPJBf1gz zw;+;ls*Y{sm9XzEW z>0|4Jg^AdmF0!t1Fx^QXnh~O4av`_ZJ=C`)WVY{kQ4!LGL5^){W zrv}L@KK#qLZp6WH_p}@C_ivDvFSKn^kym^r@K_#M3>g`e6kB3QH6`*vG$S8fuHWrq z#?5=7g0l@0Koc#2e0gt%!Y$1RR9DA?Meg{z+ayJFW6E#}2+>;?&T-~nZ_m}(hC@SE zRSM2|gznp_?+mMWS_H3&mmjtSIUQDeKFSPcq(OjAwQg*OMm8GJ^7nHwRe^Uzs_!CIm;O2 z=SYjfYCMgq%duH2lTY)fSOq?rCMz&PXsE`|a*&2C8w*^Y;H<4Qxs>G>@Y5CiL3WD` zO|;(XmA9@!N_s?8adAWa?rFkuo*G(gt`k7lPd07~T-oEGRea=1Bo3y1Bn4JIm^1sZ zL;LQ|)F|R`TKt1%E~Y!`XLctYz>M{Z3WV*6xK31DadYDIr6lEkms#7FBhO+$)}o?6 zU)jc8Z=Ahu80F5(JLD&DMRCU>2!-hPDk}pK9}D_X$4^%aN-G+Oz-!5LVDhztBA;#` zfRivKojA0_+mJ5u$zllg_+R&d;r<;*sC#-{O>{bUSNixxvZci9rgO0(-k$N4dCg0W zi~gD?V_DzdKsLM4jO^6uto{%^6OHdYI~@15Fff5J61H@A=eIHO+ljT>@~saFi%th4 z;rg)QE8E-opsrxDsxDVG>&GrBR0&7X9z4l|PcHlGrft;7dc3XgU7E*E#&?u9q=8&U zqSfh4pM$b}ySK#0*!01OCiGdTquDVtR(5LS`7Y1p{Xtsc9-D~hXHC(RM{p#b!xQaC z$%FX=B5xvR({7RmO5&p^>%tzcN*;UX7K%Hru7q%8j({v36SwWQ2h2 zHk0b=mz>f5<41qF3nNT2yb0}FYo9G&F9R+mavNmnlFfAETajFe&eKoqmmRn5P$`a> zxlUTZxpu;|_aJHHTpj%2;>wptf;iAHJoZ>%D`B>wY2YEEK67HO=r|h1 zKF(5aFC*r2C4ES&%2) zZ_W}=GCF}7yE5wTd;yRpsv!($Zo>L(X2D`Wrx(Yl8YN*yuXwf#KR{@^*qcWmef+%Rx?mG%_m=V0|<>_kQ>6U?M4oGc}kvJ%3*4 zq1!S<-7#`Z z?dW>0_qnp5wxN?u@1yZ4wnJlHW#!>fQ9Re!ClE3oV8vW8m6L2+8aKJ zbO)VGNeAd&Qt!8``S&fm)^O@8n-?A!F;=nvJn^y}*V*%QWsVJ$h4@`y@>Uk1jim-N zu(~`1>!;v@74F*=rlh{>X^X1yDW_VjpS5MeBv4@=-)Jv5sm!(@x3Y*G2>=KnpOB8(21!pD zL57Hjxyl&AI(c}_ri1qvM`#wFX z+WK$^0p9a;da^jP%B96yvtN&Tp><-uzB<<@mNjJpCG1|*vg@^Z{3XAqT! zsUL>oTf`7=v#@Ptp`#?HPUd_UkDHLYXCj4l0Xuxw#)d{rQO3;K#7VI6zjbf?Dwbow zzS*9B4j0@wTWTwRE)VHqC8G=R=(slgnrSOy^}I%_blK>F7eiw0sI+HpAv z)yRC9Mn!Ax;1BopS`U4U8v^s>nT~*^G?NL%IYO&Yt=fR?{biKpONS{X6w7Aqk<14f zC+Ve`(((f`8!M1{F)1k)NT;az>x9uiKnjvNGp09>6t;U+gCx-P`_VR#i!i5PHRSNq zz2Ub?B8yH371S1n9XiLe=}$d6J@$OAH@u*R%#FGoTB?`TUymRAv#4b+>gBVNOq}t17fqL{3w&>^WX#r8LYQ!48vN&hS4lYuLnh7S@^ZZuF0g9ac)$z;JFHd+L37 zs0Xe=`9s0U<@p>Dmm%4GJAangHqj6^mhs&MaV-zE9o^L_^J>48JTMtymV#98>o15@ zymS^?DJHtc+n&Y2GZk9m&-{#hu$^o&Ue_F^N9<$^CU{{9iovcsATbB8@n{?w`K81* zxLZ7}4Nml#VGHf;NW5mvOOv9C(uvH7NO-GcsPd9BZ(p?W$iWg`C!!mEjsycJ2To!< zBb28}GLzA3wD{d~zp|5O0%SN=_)mpX?c~4*`jwjl@<@}go4z*U(cu#=s$nH05HeaQ zv12CjFu8k+e{3y9RIvkH*y~;%YSaZMqYwiba>GCAoSEMXPZ42fABQiL>|TvL+@Bv3 ze555!a-nRs2Zh-^3}aJ9`ekZ4cO9szlY2_yrgxF&j0Exv;VpX2>b2M6Q4I*@5V||! zep#yBlxEcsx{P&q%Xwm?fULvcDb3zdlj8ljU1$vLg7Y7M65@>D+k`_rRSk0gS@z>V&_YGHZ_Mda8%Z}q=w(-aK6W#G- zo!2MvM?k_n@DU0WU7K<3g~xE|8mZn7y~ybg6^qeg9h3W*Z&Az z|6N@AUk1tkO-(sO$kTLC&bGdm<}P5Z9TQDTEmj4Vw5+6wSvqj%tX zR-qU?TR_j{kA4yTc|1z%y?*B@c7JWy^Vo=jRsDb-xbTW~Tf`VVB09Je; zz9D*VlQvcA0B*U7>BDUsEUrFD@S9%-@SytQq56xlQ?E1AFZP305ljEI36?bNqKwgE;i#o6un;IoddlUrn-WD0ss+erkojX<@}!iNI7x3qV`kEY^L_G z=E8_|aNydmI)H)Z1KM=aErP&h>y;zK-_N1@*}pyG$-E?B(08r5jJ&} zi&8pOWp^?X%>BYbLRh$KRZ)?o$Ah$BRSoKP_6SCf_A}aCAfdBe6?0Goxf^)mKiZs`g zlFMuz16u3N^2N^f4F_JL&*?-@-J_Z5%O+;J39f>J4gD_JsbI6IqVkCsxx;Y*XZr8J z2F5zY#Pgsvz|v^!<~H#HXYtVFB#mQ22ngSfuFu`<>dtkRHS`tA4_BY*1)6g) z;*#0V`R4m${NwD^u5v8RE%fkFDkvO9w{HiF^#(msi>uszo>S%2nfGS$3aM9%AvTIm z^kQzRSR2mUIdG~ZXx|S!+aSyaq6@a_MvGH}@mp!wN}j2C@ykPErPlwtu9p0W9;0L{ z{~M|$^XsB|K(#zjep%fCvWoyDogkc2L;h#kME{EceqUVqvA zZOA@1)EPYSvm1_#~0#8C*X+9M)?^jyUW=osJ} zJ>#@4U62SEB*-^oud*b7C`b`3*&cgmX8wpYWV7gHIVKTM2NF*giSL!?QZqH0@f){d z?tmSd!6-nFSXWdK4FmxuKly${APhO8+@YO&OHM=UgZ9I1WI2-E_eTv9gS6jEkie*4 zZ8CrQ)E_V|ca1;xB^eu4yu7EUS!TwLfo?&#bsS6Zxw1X!S&rP>umFf(D^-nBqi=`b zW0PRL<37onfe*BjRKuz3Jg#Tm;LJ^a7`vF;3e;=Yzkp4Fgx#fqzw7VJiK`y6zIDmW z7;v}nID!UR%q(~TO0BV_ig$IG+l6cOYt5*^Oi3n-7aYMA8&oham@R>PfQ}VOGF|e` ze}0f;8nLsIw~4!ak32BYd7|${wj?tu1pdBBYROtP769xQf|eCYcDoG=PE$uvrd_Fn9v$aX9ousSRJ5YLQnN}D7I@Zf#w5CE ztrJiq&uP2K#N-^WrP-EC{e;JL5w8g)na8K@Y_#1YUqxL_2dE$29PjME@3&7laXxNV<+O==*2XJ zHB0fjX1abI+ZG1G=Un!ezyb~u{XdJPFVJQ*Wy0ibk1ycQx2 fuse-trace: true|false extension: - direct-io: true|false + direct-io: true|false # Entry Cache configuration entry_cache: From 6539d0572f83af95224a62275ef87876430ed4cf Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 05:39:00 +0000 Subject: [PATCH 02/87] Check if CI is working --- blobfuse2-ci.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 175b6667e..b141572ae 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -86,7 +86,7 @@ jobs: - script: | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin $(go env GOPATH)/bin/golangci-lint --version - $(go env GOPATH)/bin/golangci-lint run --tests=false --build-tags ${{ parameters.tags }} --exclude-dirs test,common/stats_collector,common/stats_monitor --max-issues-per-linter=0 --exclude-files component/libfuse/libfuse2_handler_test_wrapper.go,component/libfuse/libfuse_handler_test_wrapper.go > lint.log + $(go env GOPATH)/bin/golangci-lint run --tests=false --build-tags $(tags) --exclude-dirs test,common/stats_collector,common/stats_monitor --max-issues-per-linter=0 --exclude-files component/libfuse/libfuse2_handler_test_wrapper.go,component/libfuse/libfuse_handler_test_wrapper.go > lint.log result=$(cat lint.log | wc -l) if [ $result -ne 0 ]; then echo "-----------------------------------" @@ -120,7 +120,6 @@ jobs: - script: | gofmt -s -l -d . | tee >&2 displayName: 'Go Format Check' - condition: eq(${{ parameters.skip_fmt }}, true) failOnStderr: true workingDirectory: $(WORK_DIR) From 3e26c72e23cbcfa69818341490ba1cc996eab830 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 06:21:53 +0000 Subject: [PATCH 03/87] Shift bfusemon checks from nightly to CI --- azure-pipeline-templates/bfusemon.yml | 68 +++++++++ azure-pipeline-templates/build.yml | 11 +- blobfuse2-ci.yaml | 4 + blobfuse2-gen1-artifacts.yaml | 193 -------------------------- blobfuse2-nightly.yaml | 116 ---------------- 5 files changed, 80 insertions(+), 312 deletions(-) create mode 100644 azure-pipeline-templates/bfusemon.yml delete mode 100644 blobfuse2-gen1-artifacts.yaml diff --git a/azure-pipeline-templates/bfusemon.yml b/azure-pipeline-templates/bfusemon.yml new file mode 100644 index 000000000..2808b841f --- /dev/null +++ b/azure-pipeline-templates/bfusemon.yml @@ -0,0 +1,68 @@ +parameters: + - script: | + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_hmon.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) + displayName: Create Key Config File + env: + ACCOUNT_TYPE: 'block' + NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + VERBOSE_LOG: false + USE_HTTP: false + HMON_OUTPUT: $(WORK_DIR) + continueOnError: false + + - script: cat $(BLOBFUSE2_CFG) + displayName: Print config file + + - script: + mkdir -p $(MOUNT_DIR) + mkdir -p $(TEMP_DIR) + displayName: "Create directory structure" + + - template: 'azure-pipeline-templates/mount.yml' + parameters: + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) + prefix: "Mount" + mountStep: + script: | + sudo ln -s `pwd`/bfusemon /usr/local/bin/bfusemon + ls -l /usr/local/bin/bfusemon* + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) + displayName: 'Mount Blobfuse2' + workingDirectory: $(WORK_DIR) + timeoutInMinutes: 3 + continueOnError: false + + - script: | + sudo ps -aux | grep bfusemon + displayName: 'Print bfusemon Process info' + env: + mount_dir: $(MOUNT_DIR) + + - script: | + touch $(MOUNT_DIR)/h1.txt + echo "123456" > $(MOUNT_DIR)/h2.txt + cp *.md $(MOUNT_DIR)/ + mkdir $(MOUNT_DIR)/A + mv $(MOUNT_DIR)/h1.txt $(MOUNT_DIR)/h5.txt + mv $(MOUNT_DIR)/h*.* $(MOUNT_DIR)/A + rm -rf $(MOUNT_DIR)/*.md + rm -rf $(MOUNT_DIR)/A + displayName: "Test for health monitor" + workingDirectory: $(WORK_DIR) + + - script: | + sleep 20s + cat $(WORK_DIR)/monitor_*.json + displayName: "Print bfusemon output" + workingDirectory: $(WORK_DIR) + + - template: 'azure-pipeline-templates/cleanup.yml' + parameters: + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) \ No newline at end of file diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 20ab1574e..13b45055c 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -20,6 +20,9 @@ parameters: - name: skip_azcli type: string default: "true" + - name: skip_bfusemon + type: boolean + default: true - name: proxy_address type: string default: "" @@ -95,7 +98,7 @@ steps: workingDirectory: ${{ parameters.working_directory }} displayName: "Go Get" - # Building our package + # Building blobfuse2 binary - task: Go@0 inputs: command: 'build' @@ -103,12 +106,13 @@ steps: workingDirectory: ${{ parameters.working_directory }} displayName: "Go Build" - # Building our package + # Building bfusemon binary - task: Go@0 inputs: command: 'build' arguments: "-tags ${{ parameters.tags }} -o bfusemon ./tools/health-monitor/" workingDirectory: ${{ parameters.working_directory }} + condition: eq('${{ parameters.skip_bfusemon }}', false) displayName: "Go Build bfusemon" #Verification of built binary @@ -214,5 +218,6 @@ steps: - script: | cp ./utcover.cov $(Build.ArtifactStagingDirectory)/ rm -rf ./utcover.cov + condition: eq('${{ parameters.skip_ut }}', 'false') workingDirectory: ${{ parameters.working_directory }} - displayName: "Save coverage report" + displayName: "Save coverage report" \ No newline at end of file diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index b141572ae..4ad0872f4 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -81,6 +81,7 @@ jobs: fuselib: $(fuselib) skip_azcli: "false" skip_ut: "false" + skip_bfusemon: false # Code lint checks (Static-analysis) - script: | @@ -138,6 +139,9 @@ jobs: failOnStderr: true workingDirectory: $(WORK_DIR) + # checks for bfusemon + - template: 'azure-pipeline-templates/bfusemon.yml' + - task: ComponentGovernanceComponentDetection@0 inputs: scanType: 'Register' diff --git a/blobfuse2-gen1-artifacts.yaml b/blobfuse2-gen1-artifacts.yaml deleted file mode 100644 index 828644bfe..000000000 --- a/blobfuse2-gen1-artifacts.yaml +++ /dev/null @@ -1,193 +0,0 @@ -# Pipeline to build artifact for gen1 compliant adlsgen1fuse - -# Do not trigger this pipeline automatically -trigger: none -pr: none - -stages: - - # ---------------------- Build adlsgen1fuse here and copy binary to artifact ------------------------------ - - stage: BuildAdlsGen1Fuse - jobs: - - job: AdlsGen1FuseBuild - - # Define on what all OS we want to run this Pipeline - strategy: - matrix: - Ubuntu-18: - container_image: 'ubuntu-18.04' - build_type: Release - pool: - vmImage: $(container_image) - - variables: - - group: NightlyBlobFuse - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/BlobFuse-ADLSGen1' - - steps: - - checkout: none - - # Install all dependencies - # - script: | - # sudo apt-get update --fix-missing - # sudo apt-get install pkg-config cmake libcurl4-gnutls-dev libgnutls28-dev uuid-dev libgcrypt20-dev libboost-all-dev gcc g++ -y - # sudo apt-get install libfuse-dev -y - # displayName: "Basic Tools Setup" - - # - script: | - # sudo apt update - # sudo apt upgrade - # sudo apt-get install pkg-config clang-9 -y - # curl https://sh.rustup.rs -sSf | sh -s -- -y - # rustup install nightly - # rustup install nightly-2021-09-01 - # rustup default nightly-2021-09-01-x86_64-unknown-linux-gnu - # sudo apt install llvm - # llvm-config --prefix - # displayName: "Rust Setup" - - - script: | - git clone https://$(DEVOPS_UNAME):$(DEVOPS_PWD)@dev.azure.com/azstorage/_git/BlobFuse-ADLSGen1 - displayName: "Checkout code" - workingDirectory: $(ROOT_DIR) - - # Checkout branch and list commits consumed for this build - - script: | - git checkout binaryAdd - echo "Below commits were done in last 12 hours : " `date` - git --no-pager log --since="12 hours ago" --stat - displayName: "List Commits" - workingDirectory: $(WORK_DIR) - - # Build adlsgen1fuse binary - # - script: | - # cd call-datasetfuse-gen1 - # { echo 'cargo-features = ["edition2021"]'; cat Cargo.toml; } > Cargo_new.toml - # mv Cargo_new.toml Cargo.toml - # cargo build --release - # displayName: "Build the binary" - # workingDirectory: $(WORK_DIR) - - # - script: | - # cd call-datasetfuse-gen1 - # cp ../target/release/call-gen1datasetfuse $(Build.ArtifactStagingDirectory)/adlsgen1fuse - # displayName: "Copy Artifact" - # workingDirectory: $(WORK_DIR) - - - script: | - cp adlsgen1fuse $(Build.ArtifactStagingDirectory)/adlsgen1fuse - displayName: "Copy Artifact" - workingDirectory: $(WORK_DIR) - - # Publish the Artifacts - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'rustbin' - displayName: 'Publish Artifacts for adlsgen1fuse' - condition: succeeded() - - # ---------------------- Build blobfuse and create a .deb package ------------------------------ - - stage: BuildArtifact - dependsOn: BuildAdlsGen1Fuse - condition: succeeded('BuildAdlsGen1Fuse') - - jobs: - # Ubuntu builds - - job: BlobfuseBuild - strategy: - matrix: - Ubuntu-18: - container_image: 'ubuntu-18.04' - fuselib: 'libfuse-dev' - tags: 'fuse2' - - pool: - vmImage: $(container_image) - - variables: - - group: NightlyBlobFuse - - name: root_dir - value: '$(System.DefaultWorkingDirectory)' - - name: work_dir - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - - checkout: none - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code & List commits' - workingDirectory: $(root_dir) - - # list commits from past 12hrs - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - git --no-pager log --since="12 hours ago" --stat - displayName: 'List Commits' - workingDirectory: $(work_dir) - - # install dependencies required for compiling blobfuse - - script: | - sudo apt-get update --fix-missing - sudo apt-get install ruby-dev build-essential $(fuselib) -y - sudo gem install fpm -V - displayName: "Installing Dependencies" - - # Copy adlsgen1fuse binary to blobfuse root - - task: DownloadBuildArtifacts@0 - displayName: 'Download AdlsGen1Fuse Artifacts' - inputs: - artifactName: 'rustbin' - downloadPath: $(work_dir) - - - script: | - cp ./rustbin/* ./ - rm -rf ./rustbin/* - chmod 777 ./adlsgen1fuse - ls -l ./adlsgen1fuse - displayName: "Prepare for packaging" - workingDirectory: $(work_dir) - - # build blobfuse2 and generate binary - - template: 'azure-pipeline-templates/build-release.yml' - parameters: - work_dir: $(work_dir) - root_dir: $(root_dir) - tags: $(tags) - - # place the generated binary files & any additional files in appropriate locations - - script: | - mkdir -p pkgDir/usr/bin/ - mkdir -p pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/blobfuse2 pkgDir/usr/bin/blobfuse2 - cp azure-storage-fuse/adlsgen1fuse pkgDir/usr/bin/adlsgen1fuse - cp azure-storage-fuse/setup/baseConfig.yaml pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/sampleFileCacheConfig.yaml pkgDir/usr/share/blobfuse2/ - cp azure-storage-fuse/sampleStreamingConfig.yaml pkgDir/usr/share/blobfuse2/ - mkdir -p pkgDir/etc/rsyslog.d - mkdir -p pkgDir/etc/logrotate.d - cp azure-storage-fuse/setup/11-blobfuse2.conf pkgDir/etc/rsyslog.d - cp azure-storage-fuse/setup/blobfuse2-logrotate pkgDir/etc/logrotate.d/blobfuse2 - workingDirectory: $(root_dir) - displayName: 'Accumulate pkg files' - - # using fpm tool for packaging of our binary & performing post-install operations - # for additional information about fpm refer https://fpm.readthedocs.io/en/v1.13.1/ - - script: | - fpm -s dir -t deb -n blobfuse2 -C pkgDir/ -v `./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3` -d fuse \ - --maintainer "Blobfuse v-Team " --url "https://github.com/Azure/azure-storage-fuse" \ - --description "An user-space filesystem for interacting with Azure Storage" - mv ./blobfuse2*.deb ./blobfuse2-`./pkgDir/usr/bin/blobfuse2 --version | cut -d " " -f 3`-$(container_image)-x86-64.deb - cp ./blobfuse2*.deb $(Build.ArtifactStagingDirectory) - workingDirectory: $(root_dir) - displayName: 'Make Package' - - # publishing the artifacts generated - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'blobfuse2' - displayName: 'Publish Artifacts' - - \ No newline at end of file diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 72a9f2bb8..3a89c8cf8 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -1682,119 +1682,3 @@ stages: # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) - - - stage: Healthmon - jobs: - - job: Set_1 - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - containerName: 'test-cnt-ubn-22' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - fuselib: 'libfuse3-dev' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - skip_ut: true - - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_hmon.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) - displayName: Create Key Config File - env: - ACCOUNT_TYPE: 'block' - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' - VERBOSE_LOG: false - USE_HTTP: false - HMON_OUTPUT: $(WORK_DIR) - continueOnError: false - - - script: cat $(BLOBFUSE2_CFG) - displayName: Print config file - - - script: - mkdir -p $(MOUNT_DIR) - mkdir -p $(TEMP_DIR) - displayName: "Create directory structure" - - - template: 'azure-pipeline-templates/mount.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - prefix: "Mount" - mountStep: - script: | - sudo ln -s `pwd`/bfusemon /usr/local/bin/bfusemon - ls -l /usr/local/bin/bfusemon* - $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) - displayName: 'Mount Blobfuse2' - workingDirectory: $(WORK_DIR) - timeoutInMinutes: 3 - continueOnError: false - - - script: | - sudo ps -aux | grep bfusemon - displayName: 'Print bfusemon Process info' - env: - mount_dir: $(MOUNT_DIR) - - - script: | - touch $(MOUNT_DIR)/h1.txt - echo "123456" > $(MOUNT_DIR)/h2.txt - cp *.md $(MOUNT_DIR)/ - mkdir $(MOUNT_DIR)/A - mv $(MOUNT_DIR)/h1.txt $(MOUNT_DIR)/h5.txt - mv $(MOUNT_DIR)/h*.* $(MOUNT_DIR)/A - rm -rf $(MOUNT_DIR)/*.md - rm -rf $(MOUNT_DIR)/A - displayName: "Test for health monitor" - workingDirectory: $(WORK_DIR) - - - script: | - sleep 20s - cat $(WORK_DIR)/monitor_*.json - displayName: "Print bfusemon output" - workingDirectory: $(WORK_DIR) - - - template: 'azure-pipeline-templates/cleanup.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) \ No newline at end of file From 1e75540988044e4ffd1773eff7a99b7f3f45ac6b Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 06:27:01 +0000 Subject: [PATCH 04/87] Shift bfusemon checks from nightly to CI --- azure-pipeline-templates/bfusemon.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipeline-templates/bfusemon.yml b/azure-pipeline-templates/bfusemon.yml index 2808b841f..db35624b4 100644 --- a/azure-pipeline-templates/bfusemon.yml +++ b/azure-pipeline-templates/bfusemon.yml @@ -1,4 +1,4 @@ -parameters: +steps: - script: | cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_hmon.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) @@ -21,7 +21,7 @@ parameters: mkdir -p $(TEMP_DIR) displayName: "Create directory structure" - - template: 'azure-pipeline-templates/mount.yml' + - template: 'mount.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) @@ -61,7 +61,7 @@ parameters: displayName: "Print bfusemon output" workingDirectory: $(WORK_DIR) - - template: 'azure-pipeline-templates/cleanup.yml' + - template: 'cleanup.yml' parameters: working_dir: $(WORK_DIR) mount_dir: $(MOUNT_DIR) From 9a24db1f9b3c77c7f80ae3a7c22efba2f51df66e Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 07:13:50 +0000 Subject: [PATCH 05/87] * Update github pull request template * Remove CI template --- .github/pull_request_template.md | 30 +- .../blobfuse2-ci-template.yml | 158 ------ blobfuse2-nightly.yaml | 463 +++++++++--------- 3 files changed, 249 insertions(+), 402 deletions(-) delete mode 100644 azure-pipeline-templates/blobfuse2-ci-template.yml diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 6c403d854..60d622098 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,18 +1,12 @@ -## ✅ What - - - -## 🤔 Why - - - -## 👩‍🔬 How to validate if applicable - - - -## 🔖 Related links - -- [Issues]() -- [Team thread]() -- [Documents]() -- [Email Subject] + + +- [ ] The purpose of this PR is explained in this or a referenced issue. +- [ ] Tests are included and/or updated for code changes. +- [ ] Updates to module CHANGELOG.md are included. +- [ ] License headers are included in each file. +- [ ] Need Documentation change (eg: readme.md). + diff --git a/azure-pipeline-templates/blobfuse2-ci-template.yml b/azure-pipeline-templates/blobfuse2-ci-template.yml deleted file mode 100644 index bbfbbcd4b..000000000 --- a/azure-pipeline-templates/blobfuse2-ci-template.yml +++ /dev/null @@ -1,158 +0,0 @@ -parameters: - - name: fuselib - type: string - - name: tags - type: string - - name: container - type: string - - name: skip_msi - type: string - default: "true" - - name: proxy_address - type: string - default: "" - - name: skip_fmt - type: boolean - default: true - -steps: - - checkout: self - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "./go_installer.sh" - args: "./" - displayName: "GoTool Custom Setup" - - - task: Go@0 - inputs: - command: 'get' - arguments: '-d ./...' - workingDirectory: './' - displayName: "Get Dependencies" - - # Install fuse and build the code - - script: | - sudo apt-get update --fix-missing - sudo apt-get install ${{ parameters.fuselib }} gcc -y - displayName: 'Install libfuse' - - # Install azcli - - script: | - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - az --version - displayName: 'Install Azure CLI' - - # azcli login - - script: | - az login --identity --username $(ACLI_BLOBFUSE_MSI_APP_ID) - displayName: 'Azure CLI login' - - - task: Go@0 - inputs: - command: 'build' - workingDirectory: ./ - arguments: "-tags ${{ parameters.tags }} -o blobfuse2" - displayName: "Build" - - - script: | - cnfFile=$HOME/azuretest.json - echo $cnfFile - touch $cnfFile - echo "{" > $cnfFile - echo "\"block-acct\"": "\"$(AZTEST_BLOCK_ACC_NAME)\"", >> $cnfFile - echo "\"adls-acct\"": "\"$(AZTEST_ADLS_ACC_NAME)\"", >> $cnfFile - echo "\"block-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile - echo "\"adls-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile - echo "\"block-key\"": "\"$(AZTEST_BLOCK_KEY)\"", >> $cnfFile - echo "\"adls-key\"": "\"$(AZTEST_ADLS_KEY)\"", >> $cnfFile - echo "\"block-sas\"": "\"$(AZTEST_BLOCK_SAS)\"", >> $cnfFile - echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile - echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile - echo "\"adls-sas\"": "\"$(adlsSas)\"", >> $cnfFile - echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile - echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile - echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile - echo "\"skip-msi\"": "${{ parameters.skip_msi }}", >> $cnfFile - echo "\"skip-azcli\"": "false", >> $cnfFile - echo "\"proxy-address\"": "\"${{ parameters.proxy_address }}\"" >> $cnfFile - echo "}" >> $cnfFile - cat $cnfFile - displayName: "Create Configuration File" - continueOnError: false - workingDirectory: ./ - - # Code lint checks (Static-analysis) - - script: | - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin - $(go env GOPATH)/bin/golangci-lint --version - $(go env GOPATH)/bin/golangci-lint run --tests=false --build-tags ${{ parameters.tags }} --exclude-dirs test,common/stats_collector,common/stats_monitor --max-issues-per-linter=0 --exclude-files component/libfuse/libfuse2_handler_test_wrapper.go,component/libfuse/libfuse_handler_test_wrapper.go > lint.log - result=$(cat lint.log | wc -l) - if [ $result -ne 0 ]; then - echo "-----------------------------------" - echo "Below issues are found in SA" - cat lint.log - echo "-----------------------------------" - exit 1 - else - echo "-----------------------------------" - echo "No issues are found in SA" - echo "-----------------------------------" - fi - displayName: 'Static Analysis (Lint)' - condition: always() - workingDirectory: ./ - - # Copyright checks - - script: | - result=$(grep -L -r --include \*.go "`date +%Y` Microsoft Corporation" ./ | wc -l) - if [ $result -ne 0 ]; then - exit 1 - else - echo "Copyright statements are up to date" - fi - displayName: 'Copyright check' - condition: always() - failOnStderr: true - workingDirectory: ./ - - # Go code formatting checks - - script: | - gofmt -s -l -d . | tee >&2 - displayName: 'Go Format Check' - condition: eq(${{ parameters.skip_fmt }}, true) - failOnStderr: true - workingDirectory: ./ - - # Notices files check - - script: | - ./notices_fix.sh - result=$(git diff NOTICE | wc -l) - if [ $result -ne 0 ]; then - echo "Notices needs a fix. Run ./notices_fix.sh and commit NOTICE file." - exit 1 - else - echo "Notices are up to date." - fi - displayName: 'Notice file check' - condition: always() - failOnStderr: true - workingDirectory: ./ - - # Running unit tests for fuse3 on ubn-20 - - task: Go@0 - inputs: - command: 'test' - arguments: '-v -timeout=2h ./... --tags=unittest,${{ parameters.tags }} -coverprofile utcover.cov' - workingDirectory: ./ - displayName: 'Unit tests' - continueOnError: false - - - task: ComponentGovernanceComponentDetection@0 - inputs: - scanType: 'Register' - verbosity: 'Verbose' - alertWarningLevel: 'High' - displayName: "Component governance" - condition: always() \ No newline at end of file diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 3a89c8cf8..047203501 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -1455,230 +1455,241 @@ stages: mount_dir: $(MOUNT_DIR) block_size_mb: "8" - - stage: StreamDataValidation - jobs: - # Ubuntu Tests - - job: Set_1 - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - containerName: 'test-cnt-ubn-22' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - fuselib: 'libfuse3-dev' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: skipComponentGovernanceDetection - value: true - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - skip_ut: true - - - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' - parameters: - conf_template: azure_stream.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: Stream - adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - account_type: block - account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - distro_name: $(AgentName) - quick_test: false - verbose_log: ${{ parameters.verbose_log }} - clone: true - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - - - stage: FNSDataValidation - jobs: - # Ubuntu Tests - - job: Set_1 - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-20: - AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-20' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) - fuselib: 'libfuse-dev' - tags: 'fuse2' - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - containerName: 'test-cnt-ubn-22' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - fuselib: 'libfuse3-dev' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: skipComponentGovernanceDetection - value: true - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - skip_ut: true - - - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' - parameters: - conf_template: azure_key.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: Block_Blob - adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - account_type: block - account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - distro_name: $(AgentName) - quick_test: false - verbose_log: ${{ parameters.verbose_log }} - clone: true - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - - - stage: HNSDataValidation - jobs: - # Ubuntu Tests - - job: Set_1 - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-20: - AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-20' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) - fuselib: 'libfuse-dev' - tags: 'fuse2' - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - containerName: 'test-cnt-ubn-22' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - fuselib: 'libfuse3-dev' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: skipComponentGovernanceDetection - value: true - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - skip_ut: true - - - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' - parameters: - conf_template: azure_key.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: ADLS - adls: true - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) - account_type: adls - account_endpoint: https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net - distro_name: $(AgentName) - quick_test: false - verbose_log: ${{ parameters.verbose_log }} - clone: true - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) +# Run e2e tests for file cache, stream on hns, fns + + + + + + + + + + # - stage: StreamDataValidation + # jobs: + # # Ubuntu Tests + # - job: Set_1 + # timeoutInMinutes: 300 + # strategy: + # matrix: + # Ubuntu-22: + # AgentName: 'blobfuse-ubuntu22' + # containerName: 'test-cnt-ubn-22' + # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: MOUNT_DIR + # value: '$(Pipeline.Workspace)/blob_mnt' + # - name: TEMP_DIR + # value: '$(Pipeline.Workspace)/blobfuse2_tmp' + # - name: BLOBFUSE2_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.yaml' + # - name: BLOBFUSE2_ADLS_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + # - name: skipComponentGovernanceDetection + # value: true + # - name: GOPATH + # value: '$(Pipeline.Workspace)/go' + # - name: ROOT_DIR + # value: '$(System.DefaultWorkingDirectory)' + # - name: WORK_DIR + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + # steps: + # # ------------------------------------------------------- + # # Pull and build the code + # - template: 'azure-pipeline-templates/build.yml' + # parameters: + # working_directory: $(WORK_DIR) + # root_dir: $(ROOT_DIR) + # mount_dir: $(MOUNT_DIR) + # temp_dir: $(TEMP_DIR) + # gopath: $(GOPATH) + # container: $(containerName) + # tags: $(tags) + # fuselib: $(fuselib) + # skip_ut: true + + # - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' + # parameters: + # conf_template: azure_stream.yaml + # config_file: $(BLOBFUSE2_CFG) + # container: $(containerName) + # idstring: Stream + # adls: false + # account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) + # account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) + # account_type: block + # account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net + # distro_name: $(AgentName) + # quick_test: false + # verbose_log: ${{ parameters.verbose_log }} + # clone: true + # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable + # temp_dir: $(TEMP_DIR) + # mount_dir: $(MOUNT_DIR) + + # - stage: FNSDataValidation + # jobs: + # # Ubuntu Tests + # - job: Set_1 + # timeoutInMinutes: 300 + # strategy: + # matrix: + # Ubuntu-20: + # AgentName: 'blobfuse-ubuntu20' + # containerName: 'test-cnt-ubn-20' + # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) + # fuselib: 'libfuse-dev' + # tags: 'fuse2' + # Ubuntu-22: + # AgentName: 'blobfuse-ubuntu22' + # containerName: 'test-cnt-ubn-22' + # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: MOUNT_DIR + # value: '$(Pipeline.Workspace)/blob_mnt' + # - name: TEMP_DIR + # value: '$(Pipeline.Workspace)/blobfuse2_tmp' + # - name: BLOBFUSE2_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.yaml' + # - name: BLOBFUSE2_ADLS_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + # - name: skipComponentGovernanceDetection + # value: true + # - name: GOPATH + # value: '$(Pipeline.Workspace)/go' + # - name: ROOT_DIR + # value: '$(System.DefaultWorkingDirectory)' + # - name: WORK_DIR + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + # steps: + # # ------------------------------------------------------- + # # Pull and build the code + # - template: 'azure-pipeline-templates/build.yml' + # parameters: + # working_directory: $(WORK_DIR) + # root_dir: $(ROOT_DIR) + # mount_dir: $(MOUNT_DIR) + # temp_dir: $(TEMP_DIR) + # gopath: $(GOPATH) + # container: $(containerName) + # tags: $(tags) + # fuselib: $(fuselib) + # skip_ut: true + + # - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' + # parameters: + # conf_template: azure_key.yaml + # config_file: $(BLOBFUSE2_CFG) + # container: $(containerName) + # idstring: Block_Blob + # adls: false + # account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) + # account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) + # account_type: block + # account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net + # distro_name: $(AgentName) + # quick_test: false + # verbose_log: ${{ parameters.verbose_log }} + # clone: true + # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable + # temp_dir: $(TEMP_DIR) + # mount_dir: $(MOUNT_DIR) + + # - stage: HNSDataValidation + # jobs: + # # Ubuntu Tests + # - job: Set_1 + # timeoutInMinutes: 300 + # strategy: + # matrix: + # Ubuntu-20: + # AgentName: 'blobfuse-ubuntu20' + # containerName: 'test-cnt-ubn-20' + # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) + # fuselib: 'libfuse-dev' + # tags: 'fuse2' + # Ubuntu-22: + # AgentName: 'blobfuse-ubuntu22' + # containerName: 'test-cnt-ubn-22' + # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: MOUNT_DIR + # value: '$(Pipeline.Workspace)/blob_mnt' + # - name: TEMP_DIR + # value: '$(Pipeline.Workspace)/blobfuse2_tmp' + # - name: BLOBFUSE2_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.yaml' + # - name: BLOBFUSE2_ADLS_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + # - name: skipComponentGovernanceDetection + # value: true + # - name: GOPATH + # value: '$(Pipeline.Workspace)/go' + # - name: ROOT_DIR + # value: '$(System.DefaultWorkingDirectory)' + # - name: WORK_DIR + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + # steps: + # # ------------------------------------------------------- + # # Pull and build the code + # - template: 'azure-pipeline-templates/build.yml' + # parameters: + # working_directory: $(WORK_DIR) + # root_dir: $(ROOT_DIR) + # mount_dir: $(MOUNT_DIR) + # temp_dir: $(TEMP_DIR) + # gopath: $(GOPATH) + # container: $(containerName) + # tags: $(tags) + # fuselib: $(fuselib) + # skip_ut: true + + # - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' + # parameters: + # conf_template: azure_key.yaml + # config_file: $(BLOBFUSE2_CFG) + # container: $(containerName) + # idstring: ADLS + # adls: true + # account_name: $(AZTEST_ADLS_ACC_NAME) + # account_key: $(AZTEST_ADLS_KEY) + # account_type: adls + # account_endpoint: https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net + # distro_name: $(AgentName) + # quick_test: false + # verbose_log: ${{ parameters.verbose_log }} + # clone: true + # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable + # temp_dir: $(TEMP_DIR) + # mount_dir: $(MOUNT_DIR) From d32b156a87dbbdb9b4c99a61b8e3a4bb704f3325 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 09:53:52 +0000 Subject: [PATCH 06/87] * Update Licence so that it can be recognised by github/ * Add Script to create/delete containers rather than hardcoding the container names. --- LICENSE | 43 +++++------ .../blobfuse2-data-validation.yml | 67 ----------------- azure-pipeline-templates/build.yml | 4 +- azure-pipeline-templates/container.yml | 40 ++++++++++ .../generate-container-name.yml | 7 ++ azure-pipeline-templates/huge-list-test.yml | 7 -- blobfuse2-ci.yaml | 26 +++++-- blobfuse2-data-validation.yaml | 74 ------------------- blobfuse2-nightly.yaml | 7 -- 9 files changed, 88 insertions(+), 187 deletions(-) delete mode 100644 azure-pipeline-templates/blobfuse2-data-validation.yml create mode 100644 azure-pipeline-templates/container.yml create mode 100644 azure-pipeline-templates/generate-container-name.yml delete mode 100644 blobfuse2-data-validation.yaml diff --git a/LICENSE b/LICENSE index 41eb56af9..72c2b5d92 100755 --- a/LICENSE +++ b/LICENSE @@ -1,30 +1,21 @@ - _____ _____ _____ ____ ______ _____ ------ - | | | | | | | | | | | | | - | | | | | | | | | | | | | - | --- | | | | |-----| |---- | | |-----| |----- ------ - | | | | | | | | | | | | | - | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ +The MIT License (MIT) +Copyright © Microsoft Corporation - Licensed under the MIT License . +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: - Copyright © 2020-2025 Microsoft Corporation. All rights reserved. - Author : +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. - Permission is hereby granted, free of charge, to any person obtaining a copy - of this software and associated documentation files (the "Software"), to deal - in the Software without restriction, including without limitation the rights - to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - copies of the Software, and to permit persons to whom the Software is - furnished to do so, subject to the following conditions: - - The above copyright notice and this permission notice shall be included in all - copies or substantial portions of the Software. - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/azure-pipeline-templates/blobfuse2-data-validation.yml b/azure-pipeline-templates/blobfuse2-data-validation.yml deleted file mode 100644 index cd2896e7d..000000000 --- a/azure-pipeline-templates/blobfuse2-data-validation.yml +++ /dev/null @@ -1,67 +0,0 @@ -parameters: - - name: conf_template - type: string - - name: config_file - type: string - - name: container - type: string - - name: temp_dir - type: string - - name: mount_dir - type: string - - name: idstring - type: string - - name: adls - type: boolean - - name: account_name - type: string - - name: account_key - type: string - - name: account_type - type: string - - name: account_endpoint - - name: distro_name - type: string - - name: quick_test - type: boolean - default: true - - name: verbose_log - type: boolean - default: false - - name: clone - type: boolean - default: false - - name: kversion - type: string - -steps: - - template: 'mount.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} - prefix: ${{ parameters.idstring }} - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) - - - script: | - cd ${{ parameters.mount_dir }} - wget https://cdn.kernel.org/pub/linux/kernel/v6.x/linux-${{ parameters.kversion }}.tar.xz - displayName: 'Get kernel tarfile' - - - script: | - tar -xvf {{ parameters.mount_dir }}/linux-${{ parameters.kversion }}.tar.xz - displayName: 'Untar kernel' - - - script: | - cd {{ parameters.mount_dir }}/linux-${{ parameters.kversion }} - make defconfig - make - displayName: 'Run MAKE on the kernel' - - - template: 'cleanup.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} \ No newline at end of file diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 13b45055c..50c9a8218 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -19,7 +19,7 @@ parameters: default: "true" - name: skip_azcli type: string - default: "true" + default: "false" - name: skip_bfusemon type: boolean default: true @@ -67,6 +67,8 @@ steps: else sudo apt-get install fuse3 libfuse3-dev -y fi + fusermount3 -V + fusermount -V displayName: 'Libfuse Setup' condition: eq('${{ parameters.hostedAgent }}', true) retryCountOnTaskFailure: 3 diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml new file mode 100644 index 000000000..724193c77 --- /dev/null +++ b/azure-pipeline-templates/container.yml @@ -0,0 +1,40 @@ +# Responsible for creation and deletion of container. +parameters: + - name: create_container + type: boolean + default: false + - name: delete_container + type: boolean + default: false + - name: account_name + type: string + - name: account_key + type: string + - name: container_name + type: string + + +steps: + # Creating the Container + - script: | + az --version + # Generate container Name + gen_container_name=$(tr -dc a-z0-9 Date: Thu, 13 Feb 2025 10:01:23 +0000 Subject: [PATCH 07/87] Fix typo --- blobfuse2-ci.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 4a495b948..93c532b30 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -68,7 +68,7 @@ jobs: - template: 'azure-pipeline-templates/generate-container-name.yml' # Create a Container in the storage account - - template: 'azure-pipeline-templates/container.yaml' + - template: 'azure-pipeline-templates/container.yml' parameters: create_container: true account_name: $(AZTEST_BLOCK_ACC_NAME) @@ -151,7 +151,7 @@ jobs: # delete the container # Create a Container in the storage account - - template: 'azure-pipeline-templates/container.yaml' + - template: 'azure-pipeline-templates/container.yml' parameters: delete_container: true account_name: $(AZTEST_BLOCK_ACC_NAME) From 063077046541b01194d294974c1ee3119a540df9 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 10:05:15 +0000 Subject: [PATCH 08/87] Fix typo --- azure-pipeline-templates/container.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index 724193c77..ced64ced6 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -18,11 +18,6 @@ steps: # Creating the Container - script: | az --version - # Generate container Name - gen_container_name=$(tr -dc a-z0-9 Date: Thu, 13 Feb 2025 10:12:19 +0000 Subject: [PATCH 09/87] Fix typo --- azure-pipeline-templates/generate-container-name.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/generate-container-name.yml b/azure-pipeline-templates/generate-container-name.yml index 992da76b2..bca3c52fc 100644 --- a/azure-pipeline-templates/generate-container-name.yml +++ b/azure-pipeline-templates/generate-container-name.yml @@ -2,6 +2,6 @@ steps: - script: | CONTAINER_NAME=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) echo "Generated container name: $CONTAINER_NAME" - echo "##vso[task.setvariable variable=containerName]$CONTAINER_NAME" + echo "##vso[task.setvariable variable=containerName;]$CONTAINER_NAME" echo "Assigned container name variable: $(containerName)" displayName: 'Generate random container name' From 01062f0de6c3c2b105466e6be931d50398ea09f4 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 10:30:53 +0000 Subject: [PATCH 10/87] check something --- azure-pipeline-templates/build.yml | 13 +++++++++++++ .../generate-container-name.yml | 2 +- blobfuse2-ci.yaml | 13 +------------ 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 50c9a8218..b4326bb59 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -192,6 +192,19 @@ steps: condition: eq('${{ parameters.skip_azcli }}', 'false') retryCountOnTaskFailure: 3 + # Create Container in storage Account. + # Include the template to generate the container name + - template: 'generate-container-name.yml' + + # Create a Container in the storage account + - template: 'container.yml' + parameters: + create_container: true + account_name: $(AZTEST_BLOCK_ACC_NAME) + account_key: $(AZTEST_BLOCK_KEY) + container_name: $(containerName) + + # azcli login - script: | az login --identity --username $(ACLI_BLOBFUSE_MSI_APP_ID) diff --git a/azure-pipeline-templates/generate-container-name.yml b/azure-pipeline-templates/generate-container-name.yml index bca3c52fc..caa9712b3 100644 --- a/azure-pipeline-templates/generate-container-name.yml +++ b/azure-pipeline-templates/generate-container-name.yml @@ -1,5 +1,5 @@ steps: - - script: | + - bash: | CONTAINER_NAME=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) echo "Generated container name: $CONTAINER_NAME" echo "##vso[task.setvariable variable=containerName;]$CONTAINER_NAME" diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 93c532b30..eac79b47d 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -64,17 +64,6 @@ jobs: value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' steps: - # Include the template to generate the container name - - template: 'azure-pipeline-templates/generate-container-name.yml' - - # Create a Container in the storage account - - template: 'azure-pipeline-templates/container.yml' - parameters: - create_container: true - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) - container_name: $(containerName) - # Pull and build the code, Run the Unit Tests - template: 'azure-pipeline-templates/build.yml' parameters: @@ -156,7 +145,7 @@ jobs: delete_container: true account_name: $(AZTEST_BLOCK_ACC_NAME) account_key: $(AZTEST_BLOCK_KEY) - container_name: $(containerName) + container_name: $(containerName) - task: ComponentGovernanceComponentDetection@0 inputs: From e4a57b562e279ce8b799c946bbd6ea8dec0aefda Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 11:02:33 +0000 Subject: [PATCH 11/87] check something --- azure-pipeline-templates/build.yml | 56 +++++++++---------- azure-pipeline-templates/container.yml | 2 + .../generate-container-name.yml | 2 +- 3 files changed, 29 insertions(+), 31 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index b4326bb59..cef89aacf 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -58,7 +58,7 @@ steps: echo "Released any lock if some other process has acquired" sudo dpkg --configure -a - echo "Starting Updates and Installation of Packages" + echo "****************Starting Updates and Installation of Packages**************" sudo apt-get update --fix-missing sudo apt-get install make cmake gcc g++ git parallel -y if [ "${{ parameters.tags }}" == "fuse2" ] @@ -69,6 +69,9 @@ steps: fi fusermount3 -V fusermount -V + echo "***********************Install AzCli*************************" + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version displayName: 'Libfuse Setup' condition: eq('${{ parameters.hostedAgent }}', true) retryCountOnTaskFailure: 3 @@ -150,6 +153,26 @@ steps: env: gopath: ${{ parameters.gopath }} + # Create Container in storage Account. + # Include the template to generate the container name + - template: 'generate-container-name.yml' + + # Create a Container in the block storage account + - template: 'container.yml' + parameters: + create_container: true + account_name: $(AZTEST_BLOCK_ACC_NAME) + account_key: $(AZTEST_BLOCK_KEY) + container_name: $(containerName) + + # Create a Container in the adls storage account + - template: 'container.yml' + parameters: + create_container: true + account_name: $(AZTEST_BLOCK_ACC_NAME) + account_key: $(AZTEST_BLOCK_KEY) + container_name: $(containerName) + - script: | cnfFile=$HOME/azuretest.json echo $cnfFile @@ -157,8 +180,8 @@ steps: echo "{" > $cnfFile echo "\"block-acct\"": "\"$(AZTEST_BLOCK_ACC_NAME)\"", >> $cnfFile echo "\"adls-acct\"": "\"$(AZTEST_ADLS_ACC_NAME)\"", >> $cnfFile - echo "\"block-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile - echo "\"adls-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile + echo "\"block-cont\"": "\"$(containerName)\"", >> $cnfFile + echo "\"adls-cont\"": "\"$(containerName)\"", >> $cnfFile echo "\"block-key\"": "\"$(AZTEST_BLOCK_KEY)\"", >> $cnfFile echo "\"adls-key\"": "\"$(AZTEST_ADLS_KEY)\"", >> $cnfFile echo "\"block-sas\"": "\"$(AZTEST_BLOCK_SAS)\"", >> $cnfFile @@ -178,33 +201,6 @@ steps: continueOnError: false workingDirectory: ${{ parameters.working_directory }} - # Install azcli - - script: | - sleep 5 - sudo ps -aux | grep -iE "apt" - sudo killall apt apt-get - yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock - echo "Released any lock if some other process has acquired" - sudo dpkg --configure -a - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - az --version - displayName: 'Install Azure CLI' - condition: eq('${{ parameters.skip_azcli }}', 'false') - retryCountOnTaskFailure: 3 - - # Create Container in storage Account. - # Include the template to generate the container name - - template: 'generate-container-name.yml' - - # Create a Container in the storage account - - template: 'container.yml' - parameters: - create_container: true - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) - container_name: $(containerName) - - # azcli login - script: | az login --identity --username $(ACLI_BLOBFUSE_MSI_APP_ID) diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index ced64ced6..d16c36017 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -18,6 +18,7 @@ steps: # Creating the Container - script: | az --version + echo "Creating Container: ${{ parameters.container_name }}" az storage container create --name ${{ parameters.container_name }} \ --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ @@ -28,6 +29,7 @@ steps: # Destroying the Container - script: | az --version + echo "Deleting Container: ${{ parameters.container_name }}" az storage container delete --name ${{ parameters.container_name }} \ --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ diff --git a/azure-pipeline-templates/generate-container-name.yml b/azure-pipeline-templates/generate-container-name.yml index caa9712b3..d16680e61 100644 --- a/azure-pipeline-templates/generate-container-name.yml +++ b/azure-pipeline-templates/generate-container-name.yml @@ -2,6 +2,6 @@ steps: - bash: | CONTAINER_NAME=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) echo "Generated container name: $CONTAINER_NAME" - echo "##vso[task.setvariable variable=containerName;]$CONTAINER_NAME" + echo "##vso[task.setvariable variable=containerName]$CONTAINER_NAME" echo "Assigned container name variable: $(containerName)" displayName: 'Generate random container name' From 6c4ab579b0284b8455ba32a91613e5d74a487100 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 11:11:37 +0000 Subject: [PATCH 12/87] change datalake cred --- azure-pipeline-templates/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index cef89aacf..9770ab948 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -169,8 +169,8 @@ steps: - template: 'container.yml' parameters: create_container: true - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) + account_name: $(AZTEST_ADLS_ACC_NAME) + account_key: $(AZTEST_ADLS_KEY) container_name: $(containerName) - script: | From 023ae30e3a5bdcd266f39474fc2e871415034da5 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 13:13:45 +0000 Subject: [PATCH 13/87] Add comments to the scripts --- azure-pipeline-templates/build.yml | 9 ++++++++- azure-pipeline-templates/cleanup.yml | 27 +++++++++++++++++++++++++-- blobfuse2-ci.yaml | 13 ++++++------- 3 files changed, 39 insertions(+), 10 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 9770ab948..e2a7ba57f 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -1,3 +1,10 @@ +# This step is responsible for the following behaviour: +# 1. Setup the VM all necessary packages. +# 2. Build blobfuse, bfusemon, install AzCLI +# 3. Create Temporary Containers in the storage account. +# Caution: The Deletion of the container is left to the caller of this file, caller must call the cleanup.yml with the parameter delete_containers: true +# 4. Run Unit Tests if specified by parameter skip_ut: false + parameters: - name: root_dir type: string @@ -212,7 +219,7 @@ steps: condition: eq('${{ parameters.skip_ut }}', 'false') inputs: command: 'test' - arguments: '-v -timeout=1h ./... --tags=unittest,${{ parameters.tags }} -coverprofile utcover.cov' + arguments: '-v -timeout=45m ./... --tags=unittest,${{ parameters.tags }} -coverprofile utcover.cov' workingDirectory: ${{ parameters.working_directory }} displayName: 'Unit tests' continueOnError: true diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index feb8e3581..1c23f9072 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -1,3 +1,6 @@ +# Responsible for Unmounting the blobfuse and Deletion of the Containers. +# Default behaviour on calling this script will delete the containers on the storage account. +# caution: When this is script is called for at the end of the job make sure to set the delete_containers:false. parameters: - name: working_dir type: string @@ -5,6 +8,8 @@ parameters: type: string - name: temp_dir type: string + - name: delete_containers + type: boolean steps: - script: | @@ -18,10 +23,28 @@ steps: sudo kill -9 `pidof blobfuse2` || true rm -rf ${mount_dir}/* rm -rf ${temp_dir}/* - timeoutInMinutes: 20 + timeoutInMinutes: 5 env: working_dir: ${{ parameters.working_dir }} mount_dir: ${{ parameters.mount_dir }} temp_dir: ${{ parameters.temp_dir }} displayName: 'Cleanup' - condition: always() \ No newline at end of file + condition: always() + + # delete the container in block account + - template: 'azure-pipeline-templates/container.yml' + condition: eq('${{ parameters.delete_containers }}', true) + parameters: + delete_container: true + account_name: $(AZTEST_BLOCK_ACC_NAME) + account_key: $(AZTEST_BLOCK_KEY) + container_name: $(containerName) + + # delete the container in adls account + - template: 'azure-pipeline-templates/container.yml' + condition: eq('${{ parameters.delete_containers }}', true) + parameters: + delete_container: true + account_name: $(AZTEST_ADLS_ACC_NAME) + account_key: $(AZTEST_ADLS_KEY) + container_name: $(containerName) \ No newline at end of file diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index eac79b47d..7f0f16b5c 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -138,14 +138,13 @@ jobs: # checks for bfusemon - template: 'azure-pipeline-templates/bfusemon.yml' - # delete the container - # Create a Container in the storage account - - template: 'azure-pipeline-templates/container.yml' + # CleanUp the mount and delete the containers. + - template: 'azure-pipeline-templates/cleanup.yml' parameters: - delete_container: true - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) - container_name: $(containerName) + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) + delete_containers: true - task: ComponentGovernanceComponentDetection@0 inputs: From 678d14ede2902be5b8573b0efc4a9109509b8820 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 13:17:46 +0000 Subject: [PATCH 14/87] Add comments to the scripts --- azure-pipeline-templates/cleanup.yml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index 1c23f9072..6f6589e7f 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -32,19 +32,19 @@ steps: condition: always() # delete the container in block account - - template: 'azure-pipeline-templates/container.yml' - condition: eq('${{ parameters.delete_containers }}', true) - parameters: - delete_container: true - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) - container_name: $(containerName) + - ${{ if eq('${{ parameters.delete_containers }}', true) }}: + - template: 'azure-pipeline-templates/container.yml' + parameters: + delete_container: true + account_name: $(AZTEST_BLOCK_ACC_NAME) + account_key: $(AZTEST_BLOCK_KEY) + container_name: $(containerName) # delete the container in adls account - - template: 'azure-pipeline-templates/container.yml' - condition: eq('${{ parameters.delete_containers }}', true) - parameters: - delete_container: true - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) - container_name: $(containerName) \ No newline at end of file + - ${{ if eq('${{ parameters.delete_containers }}', true) }}: + - template: 'azure-pipeline-templates/container.yml' + parameters: + delete_container: true + account_name: $(AZTEST_ADLS_ACC_NAME) + account_key: $(AZTEST_ADLS_KEY) + container_name: $(containerName) \ No newline at end of file From f08ea68b84f87e0decbb35ee31d2ad7a3721049e Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 13:24:03 +0000 Subject: [PATCH 15/87] Add comments to the scripts --- azure-pipeline-templates/build.yml | 2 +- azure-pipeline-templates/cleanup.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index e2a7ba57f..125108ab6 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -1,7 +1,7 @@ # This step is responsible for the following behaviour: # 1. Setup the VM all necessary packages. # 2. Build blobfuse, bfusemon, install AzCLI -# 3. Create Temporary Containers in the storage account. +# 3. Create Temporary Containers in the storage account with choosing random string as name and also sets the global containerName paramter to this value. # Caution: The Deletion of the container is left to the caller of this file, caller must call the cleanup.yml with the parameter delete_containers: true # 4. Run Unit Tests if specified by parameter skip_ut: false diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index 6f6589e7f..a5ecba465 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -33,7 +33,7 @@ steps: # delete the container in block account - ${{ if eq('${{ parameters.delete_containers }}', true) }}: - - template: 'azure-pipeline-templates/container.yml' + - template: 'container.yml' parameters: delete_container: true account_name: $(AZTEST_BLOCK_ACC_NAME) @@ -42,7 +42,7 @@ steps: # delete the container in adls account - ${{ if eq('${{ parameters.delete_containers }}', true) }}: - - template: 'azure-pipeline-templates/container.yml' + - template: 'container.yml' parameters: delete_container: true account_name: $(AZTEST_ADLS_ACC_NAME) From f7d27229adb6b4ba5ab9cf2727c7c9ca89289bc3 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 13:28:32 +0000 Subject: [PATCH 16/87] Add comments to the scripts --- azure-pipeline-templates/cleanup.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index a5ecba465..4d306c085 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -10,6 +10,7 @@ parameters: type: string - name: delete_containers type: boolean + default: false #todo: remove the default value so the one who writes the scripts be cautioned. steps: - script: | From d3d553723032730cbbb671c46f49031d01ade680 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 14:16:01 +0000 Subject: [PATCH 17/87] Cleanup correctly --- azure-pipeline-templates/bfusemon.yml | 5 +- azure-pipeline-templates/cleanup.yml | 40 +++++------- blobfuse2-ci.yaml | 4 +- blobfuse2-code-coverage.yaml | 94 ++------------------------- 4 files changed, 25 insertions(+), 118 deletions(-) diff --git a/azure-pipeline-templates/bfusemon.yml b/azure-pipeline-templates/bfusemon.yml index db35624b4..5cfcf6f54 100644 --- a/azure-pipeline-templates/bfusemon.yml +++ b/azure-pipeline-templates/bfusemon.yml @@ -63,6 +63,5 @@ steps: - template: 'cleanup.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) \ No newline at end of file + unmount: true + delete_containers: false \ No newline at end of file diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index 4d306c085..019d79173 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -2,35 +2,27 @@ # Default behaviour on calling this script will delete the containers on the storage account. # caution: When this is script is called for at the end of the job make sure to set the delete_containers:false. parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string + - name: unmount + type: boolean - name: delete_containers type: boolean - default: false #todo: remove the default value so the one who writes the scripts be cautioned. steps: - - script: | - ps -ef | grep blobfuse2 - df -h - displayName: 'Check process info' + - ${{ if eq('${{ parameters.unmount }}', true) }}: + - script: | + ps -ef | grep blobfuse2 + df -h + displayName: 'Check process info' - - script: | - sudo fusermount -u ${mount_dir} - sudo fusermount3 -u ${mount_dir} - sudo kill -9 `pidof blobfuse2` || true - rm -rf ${mount_dir}/* - rm -rf ${temp_dir}/* - timeoutInMinutes: 5 - env: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} - displayName: 'Cleanup' - condition: always() + - script: | + sudo fusermount -u $(MOUNT_DIR) + sudo fusermount3 -u $(MOUNT_DIR) + sudo kill -9 `pidof blobfuse2` || true + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + timeoutInMinutes: 5 + displayName: 'Cleanup' + condition: always() # delete the container in block account - ${{ if eq('${{ parameters.delete_containers }}', true) }}: diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 7f0f16b5c..d2d0ae1dd 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -141,9 +141,7 @@ jobs: # CleanUp the mount and delete the containers. - template: 'azure-pipeline-templates/cleanup.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) + unmount: false delete_containers: true - task: ComponentGovernanceComponentDetection@0 diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 6ed952f61..3a7f45f3e 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -28,15 +28,11 @@ stages: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-20' fuselib: 'libfuse-dev' - fuselib2: 'fuse' tags: 'fuse2' Ubuntu-22: AgentName: 'blobfuse-ubuntu22' - containerName: 'test-cnt-ubn-22' fuselib: 'libfuse3-dev' - fuselib2: 'fuse3' tags: 'fuse3' pool: @@ -63,7 +59,7 @@ stages: steps: # ------------------------------------------------------- - # Pull and build the code + # Pull and build the code and create the containers. - template: 'azure-pipeline-templates/build.yml' parameters: working_directory: $(WORK_DIR) @@ -71,14 +67,12 @@ stages: mount_dir: $(MOUNT_DIR) temp_dir: $(TEMP_DIR) gopath: $(GOPATH) - container: $(containerName) tags: $(tags) fuselib: $(fuselib) skip_msi: "false" - skip_azcli: "false" # ------------------------------------------------------- - # UT based code coverage test + # UT based code coverage test. - script: | go test -v -timeout=1h ./... --tags=unittest,$(tags) -covermode count -coverprofile ./blobfuse2_ut.cov workingDirectory: $(WORK_DIR) @@ -596,84 +590,8 @@ stages: displayName: "File level coverage check" condition: always() - - - - ${{ if eq(parameters.cleanup_test, true) }}: - - stage: AccountCleanup - condition: always() - jobs: - - job: Cleanup - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-20: - AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-20' - fuselib: 'libfuse3-dev' - fuselib2: 'fuse3' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' + # CleanUp the mount and delete the containers. + - template: 'azure-pipeline-templates/cleanup.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - skip_msi: "false" - skip_azcli: "true" - - # Run the block blob cleanup - - task: Go@0 - inputs: - command: 'test' - arguments: '-timeout 120m -v test/accoutcleanup/accountcleanup_test.go' - workingDirectory: $(WORK_DIR) - displayName: 'Block Blob cleanup' - timeoutInMinutes: 120 - continueOnError: true - env: - STORAGE_ACCOUNT_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STORAGE_ACCOUNT_Key: $(NIGHTLY_STO_BLOB_ACC_KEY) - - # Run the gen2 cleanup - - task: Go@0 - inputs: - command: 'test' - arguments: '-timeout 120m -v test/accoutcleanup/accountcleanup_test.go' - workingDirectory: $(WORK_DIR) - displayName: 'Gen2 cleanup' - timeoutInMinutes: 120 - continueOnError: true - env: - STORAGE_ACCOUNT_NAME: $(AZTEST_ADLS_ACC_NAME) - STORAGE_ACCOUNT_Key: $(AZTEST_ADLS_KEY) - + unmount: false + delete_containers: true From efe92fdea87b6df5e00c94048466adb71f38a842 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 14:25:08 +0000 Subject: [PATCH 18/87] Rely on global variables --- azure-pipeline-templates/bfusemon.yml | 3 --- azure-pipeline-templates/mount.yml | 22 +++++++--------------- 2 files changed, 7 insertions(+), 18 deletions(-) diff --git a/azure-pipeline-templates/bfusemon.yml b/azure-pipeline-templates/bfusemon.yml index 5cfcf6f54..0bb5b3f68 100644 --- a/azure-pipeline-templates/bfusemon.yml +++ b/azure-pipeline-templates/bfusemon.yml @@ -23,9 +23,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) prefix: "Mount" mountStep: script: | diff --git a/azure-pipeline-templates/mount.yml b/azure-pipeline-templates/mount.yml index 39d627f80..c88bac288 100755 --- a/azure-pipeline-templates/mount.yml +++ b/azure-pipeline-templates/mount.yml @@ -1,10 +1,4 @@ parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string - name: mountStep type: step - name: prefix @@ -19,9 +13,8 @@ steps: # Cleanup before mount - template: 'cleanup.yml' parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + unmount: true + delete_containers: false # Mount the container - ${{ parameters.mountStep }} @@ -31,20 +24,19 @@ steps: sleep 10s displayName: '${{ parameters.prefix }}: Waiting for mount' env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) # Check blobfuse2 is mounted or not - script: | sudo ps -aux | grep blobfuse2 displayName: 'Print Process info' env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) # Delete all files test - - script: rm -rf ${mount_dir}/* + - script: | + rm -rf $(MOUNT_DIR)/* timeoutInMinutes: 30 displayName: '${{ parameters.prefix }}: PreStart Cleanup' continueOnError: true - condition: eq( ${{ parameters.ro_mount }}, false ) - env: - mount_dir: ${{ parameters.mount_dir }} \ No newline at end of file + condition: eq( ${{ parameters.ro_mount }}, false ) \ No newline at end of file From 7b72b10a2fed696ea29a6e64b6028f1b8dfe08a3 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 14:34:26 +0000 Subject: [PATCH 19/87] Call the variable correctly --- azure-pipeline-templates/cleanup.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index 019d79173..4a03c69c5 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -8,7 +8,7 @@ parameters: type: boolean steps: - - ${{ if eq('${{ parameters.unmount }}', true) }}: + - ${{ if eq(parameters.unmount, true) }}: - script: | ps -ef | grep blobfuse2 df -h @@ -25,7 +25,7 @@ steps: condition: always() # delete the container in block account - - ${{ if eq('${{ parameters.delete_containers }}', true) }}: + - ${{ if eq(parameters.delete_containers, true) }}: - template: 'container.yml' parameters: delete_container: true @@ -34,7 +34,7 @@ steps: container_name: $(containerName) # delete the container in adls account - - ${{ if eq('${{ parameters.delete_containers }}', true) }}: + - ${{ if eq(parameters.delete_containers, true) }}: - template: 'container.yml' parameters: delete_container: true From 76f7cff62527d3207bfec79de8425dec2937f481 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Thu, 13 Feb 2025 15:01:41 +0000 Subject: [PATCH 20/87] Modify CI to not work on all the branches --- blobfuse2-ci.yaml | 6 ++++++ blobfuse2-code-coverage.yaml | 12 ------------ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index d2d0ae1dd..187b42b46 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -3,9 +3,15 @@ # Add steps that build, run tests, deploy, and more: # https://aka.ms/yaml +# Run CI on all the PR's merging to main pr: - main +# Run CI when changes occured to main branch / branch with name ci/* +trigger: +- main +- ci/* + jobs: # Ubuntu based test suite - job: test diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 3a7f45f3e..376c8af8b 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -6,18 +6,6 @@ # 3. Clear the _work or work directory which must be in myagent or $(HOME) directory # 4. Verify whether system is online from the Azure pipelines portal -parameters: - - name: coverage_test - displayName: 'Code Coverage' - type: boolean - default: true - - - name: cleanup_test - displayName: 'Account Cleanup' - type: boolean - default: true - - stages: - ${{ if eq(parameters.coverage_test, true) }}: - stage: BuildAndTest From d6d498e39a0df4011fe7beec5c2ea7c30f294c36 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 06:06:22 +0000 Subject: [PATCH 21/87] Print Type of containers when they were getting created/deleted --- azure-pipeline-templates/build.yml | 53 ++++++++++++-------------- azure-pipeline-templates/cleanup.yml | 2 + azure-pipeline-templates/container.yml | 6 ++- blobfuse2-code-coverage.yaml | 4 -- 4 files changed, 30 insertions(+), 35 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 125108ab6..11f4173d9 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -6,14 +6,6 @@ # 4. Run Unit Tests if specified by parameter skip_ut: false parameters: - - name: root_dir - type: string - - name: working_directory - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string - name: hostedAgent type: boolean default: true @@ -98,8 +90,8 @@ steps: # Custom script to install Go-lang - task: ShellScript@2 inputs: - scriptPath: "${{ parameters.working_directory }}/go_installer.sh" - args: "${{ parameters.root_dir }}/" + scriptPath: "$(WORK_DIR)/go_installer.sh" + args: "$(ROOT_DIR)/" displayName: "GoTool Custom Setup" # Downloading Go dependency packages @@ -107,7 +99,7 @@ steps: inputs: command: 'get' arguments: '-d' - workingDirectory: ${{ parameters.working_directory }} + workingDirectory: $(WORK_DIR) displayName: "Go Get" # Building blobfuse2 binary @@ -115,7 +107,7 @@ steps: inputs: command: 'build' arguments: "-tags ${{ parameters.tags }} -o blobfuse2" - workingDirectory: ${{ parameters.working_directory }} + workingDirectory: $(WORK_DIR) displayName: "Go Build" # Building bfusemon binary @@ -123,35 +115,35 @@ steps: inputs: command: 'build' arguments: "-tags ${{ parameters.tags }} -o bfusemon ./tools/health-monitor/" - workingDirectory: ${{ parameters.working_directory }} + workingDirectory: $(WORK_DIR) condition: eq('${{ parameters.skip_bfusemon }}', false) displayName: "Go Build bfusemon" #Verification of built binary - script: | - sudo chmod +x ${working_directory}/blobfuse2 - ${working_directory}/blobfuse2 --version + sudo chmod +x $(WORK_DIR)/blobfuse2 + $(WORK_DIR)/blobfuse2 --version displayName: 'Test binary' env: - working_directory: ${{ parameters.working_directory }} + working_directory: $(WORK_DIR) # Creating necessary directories - script: | - sudo fusermount -u ${mount_dir} - sudo fusermount3 -u ${mount_dir} - rm -rf ${mount_dir} - mkdir -p ${mount_dir} - echo "Creating mount dir " ${mount_dir} + sudo fusermount -u $(MOUNT_DIR) + sudo fusermount3 -u $(MOUNT_DIR) + rm -rf $(MOUNT_DIR) + mkdir -p $(MOUNT_DIR) + echo "Creating mount dir " $(MOUNT_DIR) displayName: 'Create Mount directory' env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) - script: | - mkdir -p ${temp_dir} - echo "Creating temp dir " ${temp_dir} + mkdir -p $(TEMP_DIR) + echo "Creating temp dir " $(TEMP_DIR) displayName: "Create Cache directory" env: - temp_dir: ${{ parameters.temp_dir }} + temp_dir: $(TEMP_DIR) - script: | mkdir -p ${gopath}/src @@ -162,12 +154,14 @@ steps: # Create Container in storage Account. # Include the template to generate the container name + # This will set the value generated to the global containerName parameter for the job. - template: 'generate-container-name.yml' # Create a Container in the block storage account - template: 'container.yml' parameters: create_container: true + account_type: "block" account_name: $(AZTEST_BLOCK_ACC_NAME) account_key: $(AZTEST_BLOCK_KEY) container_name: $(containerName) @@ -176,6 +170,7 @@ steps: - template: 'container.yml' parameters: create_container: true + account_type: "adls" account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) container_name: $(containerName) @@ -206,7 +201,7 @@ steps: cat $cnfFile displayName: "Create AzureTest Config" continueOnError: false - workingDirectory: ${{ parameters.working_directory }} + workingDirectory: $(WORK_DIR) # azcli login - script: | @@ -220,7 +215,7 @@ steps: inputs: command: 'test' arguments: '-v -timeout=45m ./... --tags=unittest,${{ parameters.tags }} -coverprofile utcover.cov' - workingDirectory: ${{ parameters.working_directory }} + workingDirectory: $(WORK_DIR) displayName: 'Unit tests' continueOnError: true @@ -229,7 +224,7 @@ steps: inputs: command: 'tool' arguments: 'cover -func utcover.cov' - workingDirectory: ${{ parameters.working_directory }} + workingDirectory: $(WORK_DIR) displayName: 'Code Coverage Report' continueOnError: true @@ -237,5 +232,5 @@ steps: cp ./utcover.cov $(Build.ArtifactStagingDirectory)/ rm -rf ./utcover.cov condition: eq('${{ parameters.skip_ut }}', 'false') - workingDirectory: ${{ parameters.working_directory }} + workingDirectory: $(WORK_DIR) displayName: "Save coverage report" \ No newline at end of file diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index 4a03c69c5..d80882b3c 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -29,6 +29,7 @@ steps: - template: 'container.yml' parameters: delete_container: true + account_type: "block" account_name: $(AZTEST_BLOCK_ACC_NAME) account_key: $(AZTEST_BLOCK_KEY) container_name: $(containerName) @@ -38,6 +39,7 @@ steps: - template: 'container.yml' parameters: delete_container: true + account_type: "adls" account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) container_name: $(containerName) \ No newline at end of file diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index d16c36017..43e98647c 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -6,6 +6,8 @@ parameters: - name: delete_container type: boolean default: false + - name: account_type # accepted values: block, adls + type: string - name: account_name type: string - name: account_key @@ -23,7 +25,7 @@ steps: --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ --fail-on-exist - displayName: 'Create Container' + displayName: 'Creating ${{ parameters.account_type }} Container' condition: eq('${{ parameters.create_container }}', true) # Destroying the Container @@ -33,5 +35,5 @@ steps: az storage container delete --name ${{ parameters.container_name }} \ --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ - displayName: 'Delete Container' + displayName: 'Deleting ${{ parameters.account_type }} Container' condition: eq('${{ parameters.delete_container }}', true) \ No newline at end of file diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 376c8af8b..d6165640a 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -50,10 +50,6 @@ stages: # Pull and build the code and create the containers. - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) gopath: $(GOPATH) tags: $(tags) fuselib: $(fuselib) From 22b36797f6cb2865d903d6bc1650ec222133a748 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 06:20:52 +0000 Subject: [PATCH 22/87] Remove stuff before passing --- blobfuse2-ci.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 187b42b46..6693b71f7 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -73,10 +73,6 @@ jobs: # Pull and build the code, Run the Unit Tests - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(Pipeline.Workspace) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) gopath: $(GOPATH) container: $(containerName) tags: $(tags) From 3813b78a242a2ac6c30ed8f250ff5b1a0859e1fa Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 06:34:24 +0000 Subject: [PATCH 23/87] Remove variables in codecoverage --- azure-pipeline-templates/cleanup.yml | 2 +- blobfuse2-code-coverage.yaml | 1145 +++++++++++++------------- 2 files changed, 573 insertions(+), 574 deletions(-) diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index d80882b3c..2f5c3f278 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -21,7 +21,7 @@ steps: rm -rf $(MOUNT_DIR)/* rm -rf $(TEMP_DIR)/* timeoutInMinutes: 5 - displayName: 'Cleanup' + displayName: 'Unmount Container' condition: always() # delete the container in block account diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index d6165640a..d45ab2d57 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -6,576 +6,575 @@ # 3. Clear the _work or work directory which must be in myagent or $(HOME) directory # 4. Verify whether system is online from the Azure pipelines portal -stages: - - ${{ if eq(parameters.coverage_test, true) }}: - - stage: BuildAndTest - jobs: - - job: Coverage - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-20: - AgentName: 'blobfuse-ubuntu20' - fuselib: 'libfuse-dev' - tags: 'fuse2' - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code and create the containers. - - template: 'azure-pipeline-templates/build.yml' - parameters: - gopath: $(GOPATH) - tags: $(tags) - fuselib: $(fuselib) - skip_msi: "false" - - # ------------------------------------------------------- - # UT based code coverage test. - - script: | - go test -v -timeout=1h ./... --tags=unittest,$(tags) -covermode count -coverprofile ./blobfuse2_ut.cov - workingDirectory: $(WORK_DIR) - displayName: "UT Code Coverage" - - # ------------------------------------------------------- - # Build code coverage binary - - script: | - go test -coverpkg="./..." -covermode=count -c --tags=$(tags) -o blobfuse2.test - workingDirectory: $(WORK_DIR) - displayName: "Build coverage binary" - - # ------------------------------------------------------- - # Config Generation (Block Blob) - - script: | - cd $(WORK_DIR) - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_gentest1.cov gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) - env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' - VERBOSE_LOG: false - USE_HTTP: false - displayName: 'Create Config File - Block Blob' - workingDirectory: $(WORK_DIR) - continueOnError: false - - # Code Coverage with e2e-tests for block blob - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_block.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --foreground=true & - sleep 10 - ps -aux | grep blobfuse2 - rm -rf $(MOUNT_DIR)/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) - cd - - sudo fusermount -u $(MOUNT_DIR) - sleep 5 - workingDirectory: $(WORK_DIR) - displayName: "Block Blob Coverage" - - # Code Coverage with health monitor, dynamic, CPU and mem profilers enabled for block blob - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - cp $(BLOBFUSE2_CFG) /tmp/configBlockProfilerTemp.yaml - echo "dynamic-profile: true" >> /tmp/configBlockProfilerTemp.yaml - echo "cpu-profile: /tmp/cpuBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml - echo "mem-profile: /tmp/memBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml - echo "health_monitor:" >> /tmp/configBlockProfilerTemp.yaml - echo " enable-monitoring: true" >> /tmp/configBlockProfilerTemp.yaml - echo " monitor-disable-list:" >> /tmp/configBlockProfilerTemp.yaml - echo " - blobfuse_stats" >> /tmp/configBlockProfilerTemp.yaml - cat /tmp/configBlockProfilerTemp.yaml - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_block_profiler.cov mount $(MOUNT_DIR) --config-file=/tmp/configBlockProfilerTemp.yaml --foreground=true & - sleep 10 - ps -aux | grep blobfuse2 - rm -rf $(MOUNT_DIR)/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) - cd - - sudo fusermount -u $(MOUNT_DIR) - sleep 5 - workingDirectory: $(WORK_DIR) - displayName: "Block Blob Coverage with profilers" - - # Config Generation (ADLS) - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) - env: - NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' - VERBOSE_LOG: false - USE_HTTP: false - displayName: 'Create Config File - ADLS' - continueOnError: false - - # Code Coverage with e2e-tests for ADLS - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_adls.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --foreground=true & - sleep 10 - ps -aux | grep blobfuse2 - rm -rf $(MOUNT_DIR)/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -adls=true -tmp-path=$(TEMP_DIR) - cd - - ./blobfuse2 unmount all - sleep 5 - workingDirectory: $(WORK_DIR) - displayName: "ADLS Coverage" - - # Code Coverage with health monitor, dynamic, CPU and mem profilers enabled for adls - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - cp $(BLOBFUSE2_ADLS_CFG) /tmp/configAdlsProfilerTemp.yaml - echo "dynamic-profile: true" >> /tmp/configAdlsProfilerTemp.yaml - echo "cpu-profile: /tmp/cpuAdlsProfTmp" >> /tmp/configAdlsProfilerTemp.yaml - echo "mem-profile: /tmp/memAdlsProfTmp" >> /tmp/configAdlsProfilerTemp.yaml - echo "health_monitor:" >> /tmp/configAdlsProfilerTemp.yaml - echo " enable-monitoring: true" >> /tmp/configAdlsProfilerTemp.yaml - echo " monitor-disable-list:" >> /tmp/configAdlsProfilerTemp.yaml - echo " - blobfuse_stats" >> /tmp/configAdlsProfilerTemp.yaml - cat /tmp/configAdlsProfilerTemp.yaml - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_adls_profiler.cov mount $(MOUNT_DIR) --config-file=/tmp/configAdlsProfilerTemp.yaml --foreground=true & - sleep 10 - ps -aux | grep blobfuse2 - rm -rf $(MOUNT_DIR)/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) - cd - - sudo fusermount -u $(MOUNT_DIR) - sleep 5 - workingDirectory: $(WORK_DIR) - displayName: "ADLS Coverage with profilers" - - # ------------------------------------------------------- - # Config Generation (Block Blob) for cli options - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) - displayName: 'Create Config File - Block Blob' - env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' - VERBOSE_LOG: false - USE_HTTP: false - - # Component generation code coverage - - script: | - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/generate_cmd.cov generate test_component - if [ $? -ne 0 ]; then - exit 1 - fi - workingDirectory: $(WORK_DIR) - displayName: "CLI : Component Generation" - - # Mount / Unmount / List code coverage - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/list_empty_cmd.cov mount list - if [ $? -ne 0 ]; then - exit 1 - fi - workingDirectory: $(WORK_DIR) - displayName: "CLI : Mount List" - - - script: | - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_cmd.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug - if [ $? -ne 0 ]; then - exit 1 - fi - - sleep 20 - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/list_cmd_all.cov mount list - if [ $? -ne 0 ]; then - exit 1 - fi - ./blobfuse2 unmount all - workingDirectory: $(WORK_DIR) - displayName: "CLI : Mount all and List" - - - script: | - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_cmd_all.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug - if [ $? -ne 0 ]; then - exit 1 - fi - - sleep 20 - cd $(MOUNT_DIR)/$(containerName) - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_cmd_cont_fail.cov unmount $(MOUNT_DIR)/$(containerName) - cd - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_cmd_cont.cov unmount $(MOUNT_DIR)/$(containerName) - if [ $? -ne 0 ]; then - exit 1 - fi - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_wild_cmd.cov unmount "testmut*" - if [ $? -ne 0 ]; then - exit 1 - fi - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_wild_cmd.cov unmount "abcdef*" - if [ $? -ne 0 ]; then - exit 1 - fi - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_negative_cmd.cov unmount abcdef - if [ $? -ne 1 ]; then - exit 1 - fi - - for i in {1..5}; do ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_all_cmd$i.cov unmount all; done - workingDirectory: $(WORK_DIR) - displayName: "CLI : Unmount options" - - - # Mount / Unmount Negative tests - - script: | - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_neg.cov mount all /abc --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug - if [ $? -eq 0 ]; then - exit 1 - fi - workingDirectory: $(WORK_DIR) - displayName: "CLI : Invalid mount path" - timeoutInMinutes: 2 - - - script: | - ./blobfuse2 unmount all - cp $(BLOBFUSE2_CFG) /tmp/configMountall.yaml - echo "mountall:" >> /tmp/configMountall.yaml - echo " container-allowlist:" >> /tmp/configMountall.yaml - echo " - abcd" >> /tmp/configMountall.yaml - cat /tmp/configMountall.yaml - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_encrypt_all.cov secure encrypt --config-file=/tmp/configMountall.yaml --output-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 - if [ $? -ne 0 ]; then - exit 1 - fi - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_all_cmd_secure.cov mount all $(MOUNT_DIR) --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 --log-level=log_debug --foreground=true & - if [ $? -ne 0 ]; then - exit 1 - fi - - sleep 5 - ./blobfuse2 unmount all - workingDirectory: $(WORK_DIR) - displayName: "CLI : Mount all with secure config" - timeoutInMinutes: 2 - - - script: | - ./blobfuse2 unmount all - cp $(BLOBFUSE2_CFG) /tmp/configMountall.yaml - echo "mountall:" >> /tmp/configMountall.yaml - echo " container-denylist:" >> /tmp/configMountall.yaml - echo " - abcd" >> /tmp/configMountall.yaml - cat /tmp/configMountall.yaml - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_encrypt_all2.cov secure encrypt --config-file=/tmp/configMountall.yaml --output-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 - if [ $? -ne 0 ]; then - exit 1 - fi - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_all_cmd_secure2.cov mount all $(MOUNT_DIR) --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 --log-level=log_debug --foreground=true & - if [ $? -ne 0 ]; then - exit 1 - fi - - sleep 5 - ./blobfuse2 unmount all - workingDirectory: $(WORK_DIR) - displayName: "CLI : Mount all with secure config 2" - timeoutInMinutes: 2 - - - script: | - ./blobfuse2 unmount all - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_foreg.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug --foreground=true & - if [ $? -ne 0 ]; then - exit 1 - fi - - sleep 5 - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_remount.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug - if [ $? -eq 0 ]; then - exit 1 - fi - - sleep 5 - ./blobfuse2 unmount all - workingDirectory: $(WORK_DIR) - displayName: "CLI : Remount test" - timeoutInMinutes: 2 - - # Doc generation tests - - script: | - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/doc1.cov doc - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/doc2.cov doc --output-location /notexists - touch ~/a.txt - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/doc2.cov doc --output-location ~/a.txt - rm -rf ~/a.txt - workingDirectory: $(WORK_DIR) - displayName: "CLI : doc generation" - timeoutInMinutes: 2 - - # Version check - - script: | - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/version1.cov --version - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/version2.cov version - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/version2.cov version --check - workingDirectory: $(WORK_DIR) - displayName: "CLI : version" - timeoutInMinutes: 2 - - # Simulate config change - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - ./blobfuse2 unmount all - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_foreg_2.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug --foreground=true & - if [ $? -ne 0 ]; then - exit 1 - fi - - sleep 5 - echo -e "\n\nlogging:\n level:log_debug\n type:base" >> $(BLOBFUSE2_CFG) - sed -i 's/timeout:.*/timeout: 300/' $(BLOBFUSE2_CFG) - sleep 5 - - ./blobfuse2 unmount all - workingDirectory: $(WORK_DIR) - displayName: "CLI : Config change simulator" - - # Secure Config, fine to use insecure passphrase as this is just for testing - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - ./blobfuse2 unmount all - ./blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_encrypt.cov secure encrypt --config-file=$(BLOBFUSE2_CFG) --output-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 - if [ $? -ne 0 ]; then - exit 1 - fi - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_secure.cov mount $(MOUNT_DIR) --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 & - sleep 10 - ps -aux | grep blobfuse2 - rm -rf $(MOUNT_DIR)/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -adls=false -tmp-path=$(TEMP_DIR) - cd - - - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_set.cov secure set --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 --key=logging.level --value=log_debug - ./blobfuse2 unmount all - sleep 5 - workingDirectory: $(WORK_DIR) - displayName: "CLI : Secure Config" - env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' - VERBOSE_LOG: false - USE_HTTP: false - - # stop health monitor binary associated with a given blobfuse2 pid - - script: | - ./blobfuse2 unmount all - sudo kill -9 `pidof blobfuse2.test` - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - mkdir -p $(MOUNT_DIR)/hmon_test - - sudo ln -s `pwd`/bfusemon /usr/local/bin/bfusemon - ls -l /usr/local/bin/bfusemon* - - cat /tmp/configAdlsProfilerTemp.yaml - ./blobfuse2 mount $(MOUNT_DIR)/hmon_test --config-file=/tmp/configAdlsProfilerTemp.yaml - sleep 10 - ps -aux | grep blobfuse2 - ps -aux | grep bfusemon - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/bfusemon_stop_pid.cov health-monitor stop --pid=`pidof blobfuse2` - ./blobfuse2 unmount all - workingDirectory: $(WORK_DIR) - displayName: "CLI : Health monitor stop pid" - - # stop all health monitor binaries - - script: | - ./blobfuse2 unmount all - sudo kill -9 `pidof blobfuse2.test` - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - mkdir -p $(MOUNT_DIR)/hmon_test - - cat /tmp/configAdlsProfilerTemp.yaml - ./blobfuse2 mount $(MOUNT_DIR)/hmon_test --config-file=/tmp/configAdlsProfilerTemp.yaml - sleep 10 - ps -aux | grep blobfuse2 - ps -aux | grep bfusemon - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/bfusemon_stop_all.cov health-monitor stop all - ./blobfuse2 unmount all - workingDirectory: $(WORK_DIR) - displayName: "CLI : Health monitor stop all" - - # Start Proxy - - script: | - sudo apt-get update --fix-missing -y - sudo apt remove mitmproxy -y - sudo apt-get install python3-setuptools -y - sudo apt install python3-pip -y - sudo pip3 install mitmproxy - mitmdump -q -w proxy_logs.txt & - displayName: 'Install & Start Proxy' - - # Configure Proxy cert & env - - script: | - sudo mkdir /usr/local/share/ca-certificates/extra - sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt - sudo update-ca-certificates - export https_proxy="127.0.0.1:8080" - export http_proxy="127.0.0.1:8080" - displayName: 'Configure Proxy' - - # Config Generation (Block Blob Proxy) - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) - env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' - VERBOSE_LOG: false - USE_HTTP: false - displayName: 'Create Config File - Block Blob Proxy' - workingDirectory: $(WORK_DIR) - continueOnError: false - - # Code Coverage with e2e-tests for block blob Proxy - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_block_proxy.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --disable-compression=true --foreground=true & - sleep 10 - ps -aux | grep blobfuse2 - rm -rf $(MOUNT_DIR)/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) - cd - - sudo fusermount -u $(MOUNT_DIR) - sleep 5 - workingDirectory: $(WORK_DIR) - displayName: "Block Blob Coverage Proxy" - - # Config Generation (ADLS Proxy) - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) - env: - NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' - VERBOSE_LOG: false - USE_HTTP: false - displayName: 'Create Config File - ADLS Proxy' - continueOnError: false - - # Code Coverage with e2e-tests for ADLS Proxy - - script: | - rm -rf $(MOUNT_DIR)/* - rm -rf $(TEMP_DIR)/* - ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_adls_proxy.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --foreground=true & - sleep 10 - ps -aux | grep blobfuse2 - rm -rf $(MOUNT_DIR)/* - cd test/e2e_tests - go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -adls=true -tmp-path=$(TEMP_DIR) - cd - - ./blobfuse2 unmount all - sleep 5 - workingDirectory: $(WORK_DIR) - displayName: "ADLS Coverage Proxy" - - # ------------------------------------------------------- - # Coverage report consolidation - - script: | - echo 'mode: count' > ./blobfuse2_coverage_raw.rpt - tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt - cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" | grep -v "exported/exported.go" | grep -v "component/block_cache/stream.go" > ./blobfuse2_coverage.rpt - go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt - go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html - go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html - cp ./*.rpt $(Build.ArtifactStagingDirectory)/ - cp ./*.cov $(Build.ArtifactStagingDirectory)/ - cp ./*.html $(Build.ArtifactStagingDirectory)/ - workingDirectory: $(WORK_DIR) - displayName: "Save coverage report" - - # Publish the Artifacts - - task: PublishBuildArtifacts@1 - inputs: - artifactName: 'Blobfuse2 Coverage $(tags)' - displayName: 'Publish Artifacts for blobfuse2 code coverage' - condition: succeeded() - - # Overall code coverage check - - script: | - chmod 777 ./test/scripts/coveragecheck.sh - ./test/scripts/coveragecheck.sh - workingDirectory: $(WORK_DIR) - displayName: "Overall coverage check" - - # File level code coverage check - - script: | - ./test/scripts/coveragecheck.sh file - workingDirectory: $(WORK_DIR) - displayName: "File level coverage check" - condition: always() - - # CleanUp the mount and delete the containers. - - template: 'azure-pipeline-templates/cleanup.yml' - parameters: - unmount: false - delete_containers: true +stages: + - stage: BuildAndTest + jobs: + - job: Coverage + timeoutInMinutes: 300 + strategy: + matrix: + Ubuntu-20: + AgentName: 'blobfuse-ubuntu20' + fuselib: 'libfuse-dev' + tags: 'fuse2' + Ubuntu-22: + AgentName: 'blobfuse-ubuntu22' + fuselib: 'libfuse3-dev' + tags: 'fuse3' + + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: NightlyBlobFuse + - name: MOUNT_DIR + value: '$(Pipeline.Workspace)/blob_mnt' + - name: TEMP_DIR + value: '$(Pipeline.Workspace)/blobfuse2_tmp' + - name: BLOBFUSE2_CFG + value: '$(Pipeline.Workspace)/blobfuse2.yaml' + - name: BLOBFUSE2_ADLS_CFG + value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + - name: GOPATH + value: '$(Pipeline.Workspace)/go' + - name: ROOT_DIR + value: '$(System.DefaultWorkingDirectory)' + - name: WORK_DIR + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + steps: + # ------------------------------------------------------- + # Pull and build the code and create the containers. + - template: 'azure-pipeline-templates/build.yml' + parameters: + gopath: $(GOPATH) + tags: $(tags) + fuselib: $(fuselib) + skip_msi: "false" + + # ------------------------------------------------------- + # UT based code coverage test. + - script: | + go test -v -timeout=1h ./... --tags=unittest,$(tags) -covermode count -coverprofile ./blobfuse2_ut.cov + workingDirectory: $(WORK_DIR) + displayName: "UT Code Coverage" + + # ------------------------------------------------------- + # Build code coverage binary + - script: | + go test -coverpkg="./..." -covermode=count -c --tags=$(tags) -o blobfuse2.test + workingDirectory: $(WORK_DIR) + displayName: "Build coverage binary" + + # ------------------------------------------------------- + # Config Generation (Block Blob) + - script: | + cd $(WORK_DIR) + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_gentest1.cov gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) + env: + NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + ACCOUNT_TYPE: 'block' + ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + VERBOSE_LOG: false + USE_HTTP: false + displayName: 'Create Config File - Block Blob' + workingDirectory: $(WORK_DIR) + continueOnError: false + + # Code Coverage with e2e-tests for block blob + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_block.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --foreground=true & + sleep 10 + ps -aux | grep blobfuse2 + rm -rf $(MOUNT_DIR)/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) + cd - + sudo fusermount -u $(MOUNT_DIR) + sleep 5 + workingDirectory: $(WORK_DIR) + displayName: "Block Blob Coverage" + + # Code Coverage with health monitor, dynamic, CPU and mem profilers enabled for block blob + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + cp $(BLOBFUSE2_CFG) /tmp/configBlockProfilerTemp.yaml + echo "dynamic-profile: true" >> /tmp/configBlockProfilerTemp.yaml + echo "cpu-profile: /tmp/cpuBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml + echo "mem-profile: /tmp/memBlockProfTmp" >> /tmp/configBlockProfilerTemp.yaml + echo "health_monitor:" >> /tmp/configBlockProfilerTemp.yaml + echo " enable-monitoring: true" >> /tmp/configBlockProfilerTemp.yaml + echo " monitor-disable-list:" >> /tmp/configBlockProfilerTemp.yaml + echo " - blobfuse_stats" >> /tmp/configBlockProfilerTemp.yaml + cat /tmp/configBlockProfilerTemp.yaml + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_block_profiler.cov mount $(MOUNT_DIR) --config-file=/tmp/configBlockProfilerTemp.yaml --foreground=true & + sleep 10 + ps -aux | grep blobfuse2 + rm -rf $(MOUNT_DIR)/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) + cd - + sudo fusermount -u $(MOUNT_DIR) + sleep 5 + workingDirectory: $(WORK_DIR) + displayName: "Block Blob Coverage with profilers" + + # Config Generation (ADLS) + - script: | + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) + env: + NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) + ACCOUNT_TYPE: 'adls' + ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + VERBOSE_LOG: false + USE_HTTP: false + displayName: 'Create Config File - ADLS' + continueOnError: false + + # Code Coverage with e2e-tests for ADLS + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_adls.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --foreground=true & + sleep 10 + ps -aux | grep blobfuse2 + rm -rf $(MOUNT_DIR)/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -adls=true -tmp-path=$(TEMP_DIR) + cd - + ./blobfuse2 unmount all + sleep 5 + workingDirectory: $(WORK_DIR) + displayName: "ADLS Coverage" + + # Code Coverage with health monitor, dynamic, CPU and mem profilers enabled for adls + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + cp $(BLOBFUSE2_ADLS_CFG) /tmp/configAdlsProfilerTemp.yaml + echo "dynamic-profile: true" >> /tmp/configAdlsProfilerTemp.yaml + echo "cpu-profile: /tmp/cpuAdlsProfTmp" >> /tmp/configAdlsProfilerTemp.yaml + echo "mem-profile: /tmp/memAdlsProfTmp" >> /tmp/configAdlsProfilerTemp.yaml + echo "health_monitor:" >> /tmp/configAdlsProfilerTemp.yaml + echo " enable-monitoring: true" >> /tmp/configAdlsProfilerTemp.yaml + echo " monitor-disable-list:" >> /tmp/configAdlsProfilerTemp.yaml + echo " - blobfuse_stats" >> /tmp/configAdlsProfilerTemp.yaml + cat /tmp/configAdlsProfilerTemp.yaml + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_adls_profiler.cov mount $(MOUNT_DIR) --config-file=/tmp/configAdlsProfilerTemp.yaml --foreground=true & + sleep 10 + ps -aux | grep blobfuse2 + rm -rf $(MOUNT_DIR)/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) + cd - + sudo fusermount -u $(MOUNT_DIR) + sleep 5 + workingDirectory: $(WORK_DIR) + displayName: "ADLS Coverage with profilers" + + # ------------------------------------------------------- + # Config Generation (Block Blob) for cli options + - script: | + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) + displayName: 'Create Config File - Block Blob' + env: + NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + ACCOUNT_TYPE: 'block' + ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + VERBOSE_LOG: false + USE_HTTP: false + + # Component generation code coverage + - script: | + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/generate_cmd.cov generate test_component + if [ $? -ne 0 ]; then + exit 1 + fi + workingDirectory: $(WORK_DIR) + displayName: "CLI : Component Generation" + + # Mount / Unmount / List code coverage + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/list_empty_cmd.cov mount list + if [ $? -ne 0 ]; then + exit 1 + fi + workingDirectory: $(WORK_DIR) + displayName: "CLI : Mount List" + + - script: | + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_cmd.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug + if [ $? -ne 0 ]; then + exit 1 + fi + + sleep 20 + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/list_cmd_all.cov mount list + if [ $? -ne 0 ]; then + exit 1 + fi + ./blobfuse2 unmount all + workingDirectory: $(WORK_DIR) + displayName: "CLI : Mount all and List" + + - script: | + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_cmd_all.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug + if [ $? -ne 0 ]; then + exit 1 + fi + + sleep 20 + cd $(MOUNT_DIR)/$(containerName) + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_cmd_cont_fail.cov unmount $(MOUNT_DIR)/$(containerName) + cd - + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_cmd_cont.cov unmount $(MOUNT_DIR)/$(containerName) + if [ $? -ne 0 ]; then + exit 1 + fi + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_wild_cmd.cov unmount "testmut*" + if [ $? -ne 0 ]; then + exit 1 + fi + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_wild_cmd.cov unmount "abcdef*" + if [ $? -ne 0 ]; then + exit 1 + fi + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_negative_cmd.cov unmount abcdef + if [ $? -ne 1 ]; then + exit 1 + fi + + for i in {1..5}; do ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/umnt_all_cmd$i.cov unmount all; done + workingDirectory: $(WORK_DIR) + displayName: "CLI : Unmount options" + + + # Mount / Unmount Negative tests + - script: | + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_neg.cov mount all /abc --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug + if [ $? -eq 0 ]; then + exit 1 + fi + workingDirectory: $(WORK_DIR) + displayName: "CLI : Invalid mount path" + timeoutInMinutes: 2 + + - script: | + ./blobfuse2 unmount all + cp $(BLOBFUSE2_CFG) /tmp/configMountall.yaml + echo "mountall:" >> /tmp/configMountall.yaml + echo " container-allowlist:" >> /tmp/configMountall.yaml + echo " - abcd" >> /tmp/configMountall.yaml + cat /tmp/configMountall.yaml + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_encrypt_all.cov secure encrypt --config-file=/tmp/configMountall.yaml --output-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 + if [ $? -ne 0 ]; then + exit 1 + fi + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_all_cmd_secure.cov mount all $(MOUNT_DIR) --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 --log-level=log_debug --foreground=true & + if [ $? -ne 0 ]; then + exit 1 + fi + + sleep 5 + ./blobfuse2 unmount all + workingDirectory: $(WORK_DIR) + displayName: "CLI : Mount all with secure config" + timeoutInMinutes: 2 + + - script: | + ./blobfuse2 unmount all + cp $(BLOBFUSE2_CFG) /tmp/configMountall.yaml + echo "mountall:" >> /tmp/configMountall.yaml + echo " container-denylist:" >> /tmp/configMountall.yaml + echo " - abcd" >> /tmp/configMountall.yaml + cat /tmp/configMountall.yaml + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_encrypt_all2.cov secure encrypt --config-file=/tmp/configMountall.yaml --output-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 + if [ $? -ne 0 ]; then + exit 1 + fi + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_all_cmd_secure2.cov mount all $(MOUNT_DIR) --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 --log-level=log_debug --foreground=true & + if [ $? -ne 0 ]; then + exit 1 + fi + + sleep 5 + ./blobfuse2 unmount all + workingDirectory: $(WORK_DIR) + displayName: "CLI : Mount all with secure config 2" + timeoutInMinutes: 2 + + - script: | + ./blobfuse2 unmount all + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_foreg.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug --foreground=true & + if [ $? -ne 0 ]; then + exit 1 + fi + + sleep 5 + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_remount.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug + if [ $? -eq 0 ]; then + exit 1 + fi + + sleep 5 + ./blobfuse2 unmount all + workingDirectory: $(WORK_DIR) + displayName: "CLI : Remount test" + timeoutInMinutes: 2 + + # Doc generation tests + - script: | + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/doc1.cov doc + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/doc2.cov doc --output-location /notexists + touch ~/a.txt + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/doc2.cov doc --output-location ~/a.txt + rm -rf ~/a.txt + workingDirectory: $(WORK_DIR) + displayName: "CLI : doc generation" + timeoutInMinutes: 2 + + # Version check + - script: | + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/version1.cov --version + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/version2.cov version + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/version2.cov version --check + workingDirectory: $(WORK_DIR) + displayName: "CLI : version" + timeoutInMinutes: 2 + + # Simulate config change + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + ./blobfuse2 unmount all + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_foreg_2.cov mount all $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --log-level=log_debug --foreground=true & + if [ $? -ne 0 ]; then + exit 1 + fi + + sleep 5 + echo -e "\n\nlogging:\n level:log_debug\n type:base" >> $(BLOBFUSE2_CFG) + sed -i 's/timeout:.*/timeout: 300/' $(BLOBFUSE2_CFG) + sleep 5 + + ./blobfuse2 unmount all + workingDirectory: $(WORK_DIR) + displayName: "CLI : Config change simulator" + + # Secure Config, fine to use insecure passphrase as this is just for testing + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + ./blobfuse2 unmount all + ./blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_encrypt.cov secure encrypt --config-file=$(BLOBFUSE2_CFG) --output-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 + if [ $? -ne 0 ]; then + exit 1 + fi + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/mount_secure.cov mount $(MOUNT_DIR) --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 & + sleep 10 + ps -aux | grep blobfuse2 + rm -rf $(MOUNT_DIR)/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -adls=false -tmp-path=$(TEMP_DIR) + cd - + + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/secure_set.cov secure set --config-file=$(Pipeline.Workspace)/blobfuse2.azsec --passphrase=123123123123123123123123 --key=logging.level --value=log_debug + ./blobfuse2 unmount all + sleep 5 + workingDirectory: $(WORK_DIR) + displayName: "CLI : Secure Config" + env: + NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + ACCOUNT_TYPE: 'block' + ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + VERBOSE_LOG: false + USE_HTTP: false + + # stop health monitor binary associated with a given blobfuse2 pid + - script: | + ./blobfuse2 unmount all + sudo kill -9 `pidof blobfuse2.test` + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + mkdir -p $(MOUNT_DIR)/hmon_test + + sudo ln -s `pwd`/bfusemon /usr/local/bin/bfusemon + ls -l /usr/local/bin/bfusemon* + + cat /tmp/configAdlsProfilerTemp.yaml + ./blobfuse2 mount $(MOUNT_DIR)/hmon_test --config-file=/tmp/configAdlsProfilerTemp.yaml + sleep 10 + ps -aux | grep blobfuse2 + ps -aux | grep bfusemon + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/bfusemon_stop_pid.cov health-monitor stop --pid=`pidof blobfuse2` + ./blobfuse2 unmount all + workingDirectory: $(WORK_DIR) + displayName: "CLI : Health monitor stop pid" + + # stop all health monitor binaries + - script: | + ./blobfuse2 unmount all + sudo kill -9 `pidof blobfuse2.test` + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + mkdir -p $(MOUNT_DIR)/hmon_test + + cat /tmp/configAdlsProfilerTemp.yaml + ./blobfuse2 mount $(MOUNT_DIR)/hmon_test --config-file=/tmp/configAdlsProfilerTemp.yaml + sleep 10 + ps -aux | grep blobfuse2 + ps -aux | grep bfusemon + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/bfusemon_stop_all.cov health-monitor stop all + ./blobfuse2 unmount all + workingDirectory: $(WORK_DIR) + displayName: "CLI : Health monitor stop all" + + # Start Proxy + - script: | + sudo apt-get update --fix-missing -y + sudo apt remove mitmproxy -y + sudo apt-get install python3-setuptools -y + sudo apt install python3-pip -y + sudo pip3 install mitmproxy + mitmdump -q -w proxy_logs.txt & + displayName: 'Install & Start Proxy' + + # Configure Proxy cert & env + - script: | + sudo mkdir /usr/local/share/ca-certificates/extra + sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt + sudo update-ca-certificates + export https_proxy="127.0.0.1:8080" + export http_proxy="127.0.0.1:8080" + displayName: 'Configure Proxy' + + # Config Generation (Block Blob Proxy) + - script: | + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) + env: + NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + ACCOUNT_TYPE: 'block' + ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + VERBOSE_LOG: false + USE_HTTP: false + displayName: 'Create Config File - Block Blob Proxy' + workingDirectory: $(WORK_DIR) + continueOnError: false + + # Code Coverage with e2e-tests for block blob Proxy + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_block_proxy.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) --disable-compression=true --foreground=true & + sleep 10 + ps -aux | grep blobfuse2 + rm -rf $(MOUNT_DIR)/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -tmp-path=$(TEMP_DIR) + cd - + sudo fusermount -u $(MOUNT_DIR) + sleep 5 + workingDirectory: $(WORK_DIR) + displayName: "Block Blob Coverage Proxy" + + # Config Generation (ADLS Proxy) + - script: | + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) + env: + NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) + ACCOUNT_TYPE: 'adls' + ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + VERBOSE_LOG: false + USE_HTTP: false + displayName: 'Create Config File - ADLS Proxy' + continueOnError: false + + # Code Coverage with e2e-tests for ADLS Proxy + - script: | + rm -rf $(MOUNT_DIR)/* + rm -rf $(TEMP_DIR)/* + ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_adls_proxy.cov mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_ADLS_CFG) --foreground=true & + sleep 10 + ps -aux | grep blobfuse2 + rm -rf $(MOUNT_DIR)/* + cd test/e2e_tests + go test -v -timeout=7200s ./... -args -mnt-path=$(MOUNT_DIR) -adls=true -tmp-path=$(TEMP_DIR) + cd - + ./blobfuse2 unmount all + sleep 5 + workingDirectory: $(WORK_DIR) + displayName: "ADLS Coverage Proxy" + + # ------------------------------------------------------- + # Coverage report consolidation + - script: | + echo 'mode: count' > ./blobfuse2_coverage_raw.rpt + tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt + cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" | grep -v "exported/exported.go" | grep -v "component/block_cache/stream.go" > ./blobfuse2_coverage.rpt + go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt + go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html + go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html + cp ./*.rpt $(Build.ArtifactStagingDirectory)/ + cp ./*.cov $(Build.ArtifactStagingDirectory)/ + cp ./*.html $(Build.ArtifactStagingDirectory)/ + workingDirectory: $(WORK_DIR) + displayName: "Save coverage report" + + # Publish the Artifacts + - task: PublishBuildArtifacts@1 + inputs: + artifactName: 'Blobfuse2 Coverage $(tags)' + displayName: 'Publish Artifacts for blobfuse2 code coverage' + condition: succeeded() + + # Overall code coverage check + - script: | + chmod 777 ./test/scripts/coveragecheck.sh + ./test/scripts/coveragecheck.sh + workingDirectory: $(WORK_DIR) + displayName: "Overall coverage check" + + # File level code coverage check + - script: | + ./test/scripts/coveragecheck.sh file + workingDirectory: $(WORK_DIR) + displayName: "File level coverage check" + condition: always() + + # CleanUp the mount and delete the containers. + - template: 'azure-pipeline-templates/cleanup.yml' + parameters: + unmount: false + delete_containers: true From 165dd50f3abb63c3f31ea8b5a04ec331a0feec3b Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 06:40:55 +0000 Subject: [PATCH 24/87] Remove Unnecessary variables from build.yml --- azure-pipeline-templates/build.yml | 24 +++++++----------------- blobfuse2-ci.yaml | 4 ---- blobfuse2-code-coverage.yaml | 3 --- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 11f4173d9..1bff376da 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -9,10 +9,6 @@ parameters: - name: hostedAgent type: boolean default: true - - name: gopath - type: string - - name: container - type: string - name: skip_msi type: string default: "true" @@ -28,12 +24,6 @@ parameters: - name: skip_ut type: boolean default: false - - name: tags - type: string - default: "null" - - name: fuselib - type: string - default: "libfuse3-dev" steps: - script: | @@ -60,7 +50,7 @@ steps: echo "****************Starting Updates and Installation of Packages**************" sudo apt-get update --fix-missing sudo apt-get install make cmake gcc g++ git parallel -y - if [ "${{ parameters.tags }}" == "fuse2" ] + if [ "$(tags)" == "fuse2" ] then sudo apt-get install libfuse2 libfuse-dev -y else @@ -106,7 +96,7 @@ steps: - task: Go@0 inputs: command: 'build' - arguments: "-tags ${{ parameters.tags }} -o blobfuse2" + arguments: "-tags $(tags) -o blobfuse2" workingDirectory: $(WORK_DIR) displayName: "Go Build" @@ -114,7 +104,7 @@ steps: - task: Go@0 inputs: command: 'build' - arguments: "-tags ${{ parameters.tags }} -o bfusemon ./tools/health-monitor/" + arguments: "-tags $(tags) -o bfusemon ./tools/health-monitor/" workingDirectory: $(WORK_DIR) condition: eq('${{ parameters.skip_bfusemon }}', false) displayName: "Go Build bfusemon" @@ -146,11 +136,11 @@ steps: temp_dir: $(TEMP_DIR) - script: | - mkdir -p ${gopath}/src - echo "Creating gopath " ${gopath} + mkdir -p $(GOPATH)/src + echo "Creating gopath " $(GOPATH) displayName: "Create Gopath" env: - gopath: ${{ parameters.gopath }} + gopath: $(GOPATH) # Create Container in storage Account. # Include the template to generate the container name @@ -214,7 +204,7 @@ steps: condition: eq('${{ parameters.skip_ut }}', 'false') inputs: command: 'test' - arguments: '-v -timeout=45m ./... --tags=unittest,${{ parameters.tags }} -coverprofile utcover.cov' + arguments: '-v -timeout=45m ./... --tags=unittest,$(tags) -coverprofile utcover.cov' workingDirectory: $(WORK_DIR) displayName: 'Unit tests' continueOnError: true diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 6693b71f7..59338c35c 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -73,10 +73,6 @@ jobs: # Pull and build the code, Run the Unit Tests - template: 'azure-pipeline-templates/build.yml' parameters: - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) skip_azcli: "false" skip_ut: "false" skip_bfusemon: false diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index d45ab2d57..96f99bc41 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -49,9 +49,6 @@ stages: # Pull and build the code and create the containers. - template: 'azure-pipeline-templates/build.yml' parameters: - gopath: $(GOPATH) - tags: $(tags) - fuselib: $(fuselib) skip_msi: "false" # ------------------------------------------------------- From 4904cd81ef6309d7527111419d5366a1fb644921 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 11:00:16 +0000 Subject: [PATCH 25/87] Modifications start for nytly --- azure-pipeline-templates/build.yml | 66 +- azure-pipeline-templates/distro-tests.yml | 40 +- .../invalid-command-tests.yml | 42 +- azure-pipeline-templates/package-install.yml | 109 +++ azure-pipeline-templates/verbose-tests.yml | 130 ++- azure-pipeline-templates/verify-auth.yml | 34 +- blobfuse2-code-coverage.yaml | 1 + blobfuse2-nightly.yaml | 854 +++--------------- 8 files changed, 386 insertions(+), 890 deletions(-) create mode 100644 azure-pipeline-templates/package-install.yml diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 1bff376da..2b6458d14 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -6,9 +6,12 @@ # 4. Run Unit Tests if specified by parameter skip_ut: false parameters: - - name: hostedAgent - type: boolean - default: true + - name: distro_name + type: string + default: "UBUNTU" + - name: distro_id + type: string + default: "0" - name: skip_msi type: string default: "true" @@ -34,36 +37,33 @@ steps: displayName: 'Create Directory Structure' # Package manager installs for libfuse, - # Before installation, stop any process such as update manager from using package manager. - # otherwise installation may fail - - script: | - sleep 30 - sudo systemctl stop apt-daily.timer - echo "Add Lock Timeout for apt package manager" - sudo sh -c 'echo "DPkg::Lock::Timeout \"120\";" > /etc/apt/apt.conf.d/99timeout' - sudo ps -aux | grep -iE "apt" - sudo killall apt apt-get - yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock - echo "Released any lock if some other process has acquired" - sudo dpkg --configure -a - - echo "****************Starting Updates and Installation of Packages**************" - sudo apt-get update --fix-missing - sudo apt-get install make cmake gcc g++ git parallel -y - if [ "$(tags)" == "fuse2" ] - then - sudo apt-get install libfuse2 libfuse-dev -y - else - sudo apt-get install fuse3 libfuse3-dev -y - fi - fusermount3 -V - fusermount -V - echo "***********************Install AzCli*************************" - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - az --version - displayName: 'Libfuse Setup' - condition: eq('${{ parameters.hostedAgent }}', true) - retryCountOnTaskFailure: 3 + - template: 'package-install.yml' + parameters: + distro_name: $(paramters.distro_name) + distro_id: $(paramters.distro_id) + + + # If proxy is given, Install and setup the proxy, these tests can only be done on ubuntu + - ${{ if ne(parameters.proxy_address, "") }}: + # Start Proxy + - script: | + sudo apt-get update --fix-missing + sudo apt remove mitmproxy -y + sudo apt-get install python3-setuptools -y + sudo apt install python3-pip -y + sudo pip3 install werkzeug==2.3.7 + sudo pip3 install mitmproxy + mitmdump -q -w proxy_logs.txt & + displayName: 'Install & Start Proxy' + + # Configure Proxy cert & env + - script: | + sudo mkdir /usr/local/share/ca-certificates/extra + sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt + sudo update-ca-certificates + export https_proxy="127.0.0.1:8080" + export http_proxy="127.0.0.1:8080" + displayName: 'Configure Proxy' # Clone the repo - script: | diff --git a/azure-pipeline-templates/distro-tests.yml b/azure-pipeline-templates/distro-tests.yml index a77554d08..c108c1307 100755 --- a/azure-pipeline-templates/distro-tests.yml +++ b/azure-pipeline-templates/distro-tests.yml @@ -1,20 +1,6 @@ parameters: - - name: working_dir - type: string - - name: root_dir - type: string - - name: temp_dir - type: string - - name: mount_dir - type: string - name: config_path type: string - - name: container - type: string - - name: installStep - type: step - - name: gopath - type: string - name: blob_account_name type: string - name: blob_account_key @@ -24,13 +10,7 @@ parameters: - name: adls_account_key type: string - name: distro_name - type: string - - name: tags - type: string - default: "null" - - name: fuselib - type: string - default: "fuse3 libfuse3-dev" + type: string - name: quick_test type: boolean default: true @@ -47,9 +27,6 @@ steps: echo $(Description) hostnamectl displayName: 'Print Agent Info' - - # Create directory structure and prepare to mount - - ${{ parameters.installStep }} - checkout: none @@ -64,15 +41,8 @@ steps: # Build the code - template: 'build.yml' parameters: - root_dir: ${{ parameters.root_dir }} - working_directory: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} - hostedAgent: false - gopath: ${{ parameters.gopath }} - container: ${{ parameters.container }} - tags: ${{ parameters.tags }} - fuselib: ${{ parameters.fuselib }} + distro_name: $(DistroVer) + distro_id: $(distroId) # Block Blob Test - template: 'e2e-tests-spcl.yml' @@ -90,7 +60,6 @@ steps: quick_test: ${{ parameters.quick_test }} verbose_log: ${{ parameters.verbose_log }} clone: ${{ parameters.clone }} - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) @@ -110,7 +79,6 @@ steps: quick_test: ${{ parameters.quick_test }} verbose_log: ${{ parameters.verbose_log }} clone: ${{ parameters.clone }} - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) @@ -119,7 +87,7 @@ steps: inputs: command: 'clean' arguments: '-cache -modcache -testcache -i' - workingDirectory: ${{ parameters.working_dir }} + workingDirectory: $(WORK_DIR) displayName: 'Clean up go tool dirs' timeoutInMinutes: 20 continueOnError: true diff --git a/azure-pipeline-templates/invalid-command-tests.yml b/azure-pipeline-templates/invalid-command-tests.yml index ceb6aa054..0b3d84a01 100644 --- a/azure-pipeline-templates/invalid-command-tests.yml +++ b/azure-pipeline-templates/invalid-command-tests.yml @@ -1,72 +1,66 @@ -parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - steps: - script: | - ${working_dir}/blobfuse2 --invalid-param & + $(WORK_DIR)/blobfuse2 --invalid-param & if [ $? = 2 ]; then exit 0 fi displayName: 'Invalid Flag Root Command' continueOnError: false env: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) - script: | - ${working_dir}/blobfuse2 mount ${mount_dir} --invalid-param & + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --invalid-param & if [ $? = 2 ]; then exit 0 fi displayName: 'Invalid Flag Mount Command' continueOnError: false env: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) - script: | - ${working_dir}/blobfuse2 unmount ${mount_dir} --invalid-param & + $(WORK_DIR)/blobfuse2 unmount $(MOUNT_DIR) --invalid-param & if [ $? = 2 ]; then exit 0 fi displayName: 'Invalid Flag Unmount Command' continueOnError: false env: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) - script: | - ${working_dir}/blobfuse2 mountv1 ${mount_dir} --invalid-param & + $(WORK_DIR)/blobfuse2 mountv1 $(MOUNT_DIR) --invalid-param & if [ $? = 2 ]; then exit 0 fi displayName: 'Invalid Flag Mountv1 Command' continueOnError: false env: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) - script: | - ${working_dir}/blobfuse2 secure ${mount_dir} --invalid-param & + $(WORK_DIR)/blobfuse2 secure $(MOUNT_DIR) --invalid-param & if [ $? = 2 ]; then exit 0 fi displayName: 'Invalid Flag Secure Command' continueOnError: false env: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) - script: | - ${working_dir}/blobfuse2 version ${mount_dir} --invalid-param & + $(WORK_DIR)/blobfuse2 version $(MOUNT_DIR) --invalid-param & if [ $? = 2 ]; then exit 0 fi displayName: 'Invalid Flag Version Command' continueOnError: false env: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} \ No newline at end of file + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) \ No newline at end of file diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml new file mode 100644 index 000000000..022b323ad --- /dev/null +++ b/azure-pipeline-templates/package-install.yml @@ -0,0 +1,109 @@ +# If you want to add new disto, create the next available distro_id +# Currently, Supported distros, distro Id +# UBUNTU 0 +# RHEL 1 +# CENTOS 2 +# ORACLE 3 +# ROCKY 4 +# SUSE 5 +# MARINER 6 +parameters: + - name: distro_name + type: string + default: "UBUNTU" + - name: distro_id + type: string + default: "0" + + # UBUNTU PACKAGE INSTALLATION + # Before installation, stop any process such as update manager from using package manager. + # otherwise installation may fail + - script: | + sleep 30 + sudo systemctl stop apt-daily.timer + echo "Add Lock Timeout for apt package manager" + sudo sh -c 'echo "DPkg::Lock::Timeout \"120\";" > /etc/apt/apt.conf.d/99timeout' + sudo ps -aux | grep -iE "apt" + sudo killall apt apt-get + yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock + echo "Released any lock if some other process has acquired" + sudo dpkg --configure -a + + echo "****************Starting Updates and Installation of Packages**************" + sudo apt-get update --fix-missing + sudo apt-get install make cmake gcc g++ git parallel -y + if [ "$(tags)" == "fuse2" ] + then + sudo apt-get install libfuse2 libfuse-dev -y + else + sudo apt-get install fuse3 libfuse3-dev -y + fi + fusermount3 -V + fusermount -V + echo "***********************Install AzCli*************************" + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version + condition: eq('${{ parameters.distro_id }}', "0") + displayName: 'Libfuse Setup ${{ parameters.distro_name }}' + + # RHEL PACKAGE INSTALLATION + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo yum update -y + sudo yum groupinstall "Development Tools" -y + if [ $(AgentName) == "blobfuse-rhel9" ]; then + sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing + else + sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing + fi + condition: eq('${{ parameters.distro_id }}', "1") + displayName: 'Libfuse Setup ${{ parameters.distro_name }}' + + # CENTOS PACKAGE INSTALLATION + - script: | + if [ "$(AgentName)" == "blobfuse-centos8" ] + then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + sudo yum update -y --skip-broken + if [ $(AgentName) == "blobfuse-centos8" ]; then + sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing + else + sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y + fi + condition: eq('${{ parameters.distro_id }}', "2") + displayName: 'Libfuse Setup ${{ parameters.distro_name }}' + + # ORACLE PACKAGE INSTALLATION + - script: | + sudo yum update -y + sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing + condition: eq('${{ parameters.distro_id }}', "3") + displayName: 'Libfuse Setup ${{ parameters.distro_name }}' + + # ROCKY PACKAGE INSTALLATION + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo yum update -y + sudo yum groupinstall "Development Tools" -y + sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing + condition: eq('${{ parameters.distro_id }}', "4") + displayName: 'Libfuse Setup ${{ parameters.distro_name }}' + + # SUSE PACKAGE INSTALLATION + - script: | + sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse + wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm + sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm + fusermount3 --v + wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm + sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm + condition: eq('${{ parameters.distro_id }}', "5") + displayName: 'Libfuse Setup ${{ parameters.distro_name }}' + + # MARINER PACKAGE INSTALLATION + - script: | + sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y + condition: eq('${{ parameters.distro_id }}', "6") + displayName: 'Libfuse Setup ${{ parameters.distro_name }}' diff --git a/azure-pipeline-templates/verbose-tests.yml b/azure-pipeline-templates/verbose-tests.yml index 901311e5f..13675805a 100644 --- a/azure-pipeline-templates/verbose-tests.yml +++ b/azure-pipeline-templates/verbose-tests.yml @@ -19,16 +19,8 @@ parameters: type: string - name: client_secret type: string - - name: container - type: string - name: config type: string - - name: working_dir - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string - name: stress_dir type: string - name: huge_container @@ -58,15 +50,13 @@ parameters: default: "true" - name: verbose_log type: boolean - - name: tags - type: string #--------------------------------------- Setup: End to end tests with different Storage configurations ------------------------------------------ # Create key credential config file if we need to test it steps: - script: | - cd ${{ parameters.working_dir }} - ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config }} + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config }} displayName: Create Key Config File env: ACCOUNT_TYPE: ${{ parameters.account_type }} @@ -84,8 +74,8 @@ steps: # Create sas credential config file if we need to test it - script: | - cd ${{ parameters.working_dir }} - ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_sas.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.sas_credential_config }} + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_sas.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.sas_credential_config }} displayName: Create SAS Config File env: NIGHTLY_STO_BLOB_ACC_NAME: ${{ parameters.account_name }} @@ -102,8 +92,8 @@ steps: # Create azcli credential config file if we need to test it - script: | - cd ${{ parameters.working_dir }} - ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_cli.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.azcli_credential_config }} + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_cli.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.azcli_credential_config }} displayName: Create Azure CLI Config File env: NIGHTLY_STO_BLOB_ACC_NAME: ${{ parameters.account_name }} @@ -119,8 +109,8 @@ steps: # Create azurite config file if we need to test it - script: | - cd ${{ parameters.working_dir }} - ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.azurite_config }} + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.azurite_config }} displayName: Create Key Config File # Note: It is fine to hardcode these credentials since they are publicly documented fake credentials for Azurite env: @@ -142,9 +132,9 @@ steps: - ${{ if eq(parameters.test_key_credential, true) }}: - template: e2e-tests.yml parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with Key Credentials distro_name: ${{ parameters.distro_name }} @@ -153,8 +143,8 @@ steps: verbose_log: ${{ parameters.verbose_log }} mountStep: script: > - ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config }} - --default-working-dir=${{ parameters.working_dir }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config }} + --default-working-dir=$(WORK_DIR) displayName: 'E2E Test: Mount with Key Credential Configuration' timeoutInMinutes: 3 continueOnError: false @@ -165,9 +155,9 @@ steps: parameters: conf_template: azure_block_perf.yaml config_file: ${{ parameters.config }} - container: ${{ parameters.container }} - temp_dir: ${{ parameters.temp_dir }} - mount_dir: ${{ parameters.mount_dir }} + container: $(containerName) + temp_dir: $(TEMP_DIR) + mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -182,9 +172,9 @@ steps: - ${{ if eq(parameters.test_sas_credential, true) }}: - template: e2e-tests.yml parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with SAS Credentials distro_name: ${{ parameters.distro_name }} @@ -192,8 +182,8 @@ steps: verbose_log: ${{ parameters.verbose_log }} mountStep: script: > - ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.sas_credential_config }} - --default-working-dir=${{ parameters.working_dir }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.sas_credential_config }} + --default-working-dir=$(WORK_DIR) displayName: 'E2E Test: Mount with SAS Credential Configuration' timeoutInMinutes: 3 continueOnError: false @@ -201,9 +191,9 @@ steps: - ${{ if eq(parameters.test_azcli_credential, true) }}: - template: e2e-tests.yml parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with Azure CLI Credentials distro_name: ${{ parameters.distro_name }} @@ -211,8 +201,8 @@ steps: verbose_log: ${{ parameters.verbose_log }} mountStep: script: > - ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.azcli_credential_config }} - --default-working-dir=${{ parameters.working_dir }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.azcli_credential_config }} + --default-working-dir=$(WORK_DIR) displayName: 'E2E Test: Mount with Azure CLI Credential Configuration' timeoutInMinutes: 3 continueOnError: false @@ -223,14 +213,14 @@ steps: sudo npm install -g azurite@3.29.0 sudo mkdir azurite sudo azurite --silent --location azurite --debug azurite\debug.log & - az storage container create -n ${{ parameters.container }} --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" + az storage container create -n $(containerName) --connection-string "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;QueueEndpoint=http://127.0.0.1:10001/devstoreaccount1;" displayName: 'Install and Run Azurite' - template: e2e-tests.yml parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with Azurite distro_name: ${{ parameters.distro_name }} @@ -239,8 +229,8 @@ steps: verbose_log: ${{ parameters.verbose_log }} mountStep: script: > - ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.azurite_config }} - --default-working-dir=${{ parameters.working_dir }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.azurite_config }} + --default-working-dir=$(WORK_DIR) displayName: 'E2E Test: Mount with Azurite Configuration' timeoutInMinutes: 3 continueOnError: false @@ -251,9 +241,9 @@ steps: parameters: conf_template: azure_key_lru_purge.yaml config_file: ${{ parameters.config }} - container: ${{ parameters.container }} - temp_dir: ${{ parameters.temp_dir }} - mount_dir: ${{ parameters.mount_dir }} + container: $(containerName) + temp_dir: $(TEMP_DIR) + mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -267,9 +257,9 @@ steps: parameters: conf_template: azure_key_emptyfile.yaml config_file: ${{ parameters.config }} - container: ${{ parameters.container }} - temp_dir: ${{ parameters.temp_dir }} - mount_dir: ${{ parameters.mount_dir }} + container: $(containerName) + temp_dir: $(TEMP_DIR) + mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -285,9 +275,9 @@ steps: parameters: conf_template: azure_key_directio.yaml config_file: ${{ parameters.config }} - container: ${{ parameters.container }} - temp_dir: ${{ parameters.temp_dir }} - mount_dir: ${{ parameters.mount_dir }} + container: $(containerName) + temp_dir: $(TEMP_DIR) + mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -303,9 +293,9 @@ steps: parameters: conf_template: azure_key_symlink.yaml config_file: ${{ parameters.config }} - container: ${{ parameters.container }} - temp_dir: ${{ parameters.temp_dir }} - mount_dir: ${{ parameters.mount_dir }} + container: $(containerName) + temp_dir: $(TEMP_DIR) + mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -319,8 +309,8 @@ steps: #--------------------------------------- Setup: End to end tests with different File Cache configurations ------------------------------------------ - script: | - cd ${{ parameters.working_dir }} - ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config }} + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config }} displayName: Create Config File env: NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} @@ -335,32 +325,32 @@ steps: - template: mount-test.yml parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) config: ${{ parameters.config }} idstring: ${{ parameters.service }} Mount Test tags: $(tags) - template: stress-test.yml parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) stress_dir: ${{ parameters.stress_dir }} idstring: ${{ parameters.service }} Stress Test quick: '${{ parameters.quick_stress }}' distro_name: ${{ parameters.distro_name }} mountStep: script: > - ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config }} --default-working-dir=${{ parameters.working_dir }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config }} --default-working-dir=$(WORK_DIR) displayName: 'StressTest: Mount' timeoutInMinutes: 3 continueOnError: false - script: | - cd ${{ parameters.working_dir }} - ${{ parameters.working_dir }}/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=${{ parameters.huge_container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config }} + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=${{ parameters.huge_container }} --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config }} displayName: Create Config File env: NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} @@ -375,14 +365,14 @@ steps: - template: huge-list-test.yml parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) idstring: ${{ parameters.service }} Huge List Test distro_name: ${{ parameters.distro_name }} mountStep: script: > - ${{ parameters.working_dir }}/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config }} - --default-working-dir=${{ parameters.working_dir }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config }} + --default-working-dir=$(WORK_DIR) displayName: 'HugeList: Mount' continueOnError: false diff --git a/azure-pipeline-templates/verify-auth.yml b/azure-pipeline-templates/verify-auth.yml index f82389681..b6e3b63ae 100755 --- a/azure-pipeline-templates/verify-auth.yml +++ b/azure-pipeline-templates/verify-auth.yml @@ -1,10 +1,4 @@ parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string - name: mountStep type: step - name: idstring @@ -16,9 +10,9 @@ steps: # Mount the container - template: 'mount.yml' parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) prefix: ${{ parameters.idstring }} mountStep: ${{ parameters.mountStep }} @@ -26,32 +20,32 @@ steps: - script: | df sudo ps -aux | grep blobfuse2 - ls -l ${mount_dir} + ls -l $(MOUNT_DIR) displayName: 'AuthVerify: ${{ parameters.idstring }} list' continueOnError: true env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) # Run basic test case to validate auth is allowing the operations - script: | - rm -rf ${mount_dir}/* - mkdir ${mount_dir}/A - touch ${mount_dir}/A/a.txt - echo "ASDF" > ${mount_dir}/A/b.txt - cp ${mount_dir}/A/b.txt ${mount_dir}/A/c.txt - ls -lRt ${mount_dir}/A + rm -rf $(MOUNT_DIR)/* + mkdir $(MOUNT_DIR)/A + touch $(MOUNT_DIR)/A/a.txt + echo "ASDF" > $(MOUNT_DIR)/A/b.txt + cp $(MOUNT_DIR)/A/b.txt $(MOUNT_DIR)/A/c.txt + ls -lRt $(MOUNT_DIR)/A displayName: 'AuthVerify: ${{ parameters.idstring }} fileops' continueOnError: true env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) # Delete the directories - script: | - rm -rf ${mount_dir}/A + rm -rf $(MOUNT_DIR)/A displayName: 'AuthVerify: ${{ parameters.idstring }} remove' continueOnError: true env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) - script: | cat blobfuse2-logs.txt diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 96f99bc41..67488af6b 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -575,3 +575,4 @@ stages: parameters: unmount: false delete_containers: true + condition: always() diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index aadbacd19..de664ce2d 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -57,7 +57,7 @@ stages: - ${{ if eq(parameters.base_test, true) }}: - stage: BuildAndTest jobs: - # Ubuntu Tests + # Ubuntu Tests for Block account - job: Set_1 timeoutInMinutes: 300 @@ -65,124 +65,25 @@ stages: matrix: Ubuntu-20-BlockBlob: AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-20' + poolName: 'blobfuse-ubuntu-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22-BlockBlob: AgentName: 'blobfuse-ubuntu22' - containerName: 'test-cnt-ubn-22' + poolName: 'blobfuse-ubuntu-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_SAS_CFG - value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml' - - name: BLOBFUSE2_AZCLI_CFG - value: '$(Pipeline.Workspace)/blobfuse2_azcli_config.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: BLOBFUSE2_GTEST_CFG - value: '$(Pipeline.Workspace)/connection.yaml' - - name: BLOBFUSE2_AZURITE_CFG - value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml' - - name: BLOBFUSE2_STRESS_DIR - value: '$(Pipeline.Workspace)/blobfuse2_stress' - - name: DECODE_PERCENTS - value: false - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - working_directory: $(WORK_DIR) - root_dir: $(Pipeline.Workspace) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - skip_azcli: "false" - - # ------------------------------------------------------- - - template: 'azure-pipeline-templates/invalid-command-tests.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - - - ${{ if eq(parameters.exhaustive_test, true) }}: - - template: 'azure-pipeline-templates/verbose-tests.yml' - parameters: - service: 'BlockBlob' - account_type: 'block' - account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' - adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - account_sas: $(NIGHTLY_STO_ACC_SAS) - client_id: $(AZTEST_CLIENT) - tenant_id: $(AZTEST_TENANT) - client_secret: $(AZTEST_SECRET) - container: $(containerName) - config: $(BLOBFUSE2_CFG) - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - stress_dir: $(BLOBFUSE2_STRESS_DIR) - huge_container: 'testcnt1' - quick_stress: ${{ parameters.quick_stress }} - test_key_credential: true - test_sas_credential: true - test_azcli_credential: true - test_azurite: false - sas_credential_config: $(BLOBFUSE2_SAS_CFG) - azcli_credential_config: $(BLOBFUSE2_AZCLI_CFG) - azurite_config: $(BLOBFUSE2_AZURITE_CFG) - distro_name: $(AgentName) - verbose_log: ${{ parameters.verbose_log }} - tags: $(tags) - - - template: azure-pipeline-templates/cleanup.yml - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - - - job: Set_1_ARM - timeoutInMinutes: 300 - - strategy: - matrix: Ubuntu-22-ARM64-BlockBlob: AgentName: 'blobfuse-ubn22-arm64' - containerName: 'test-cnt-ubn-22-arm64' + poolName: 'blobfuse-ubn-arm64-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM) fuselib: 'libfuse3-dev' tags: 'fuse3' - + pool: - name: "blobfuse-ubn-arm64-pool" + name: $(poolName) demands: - ImageOverride -equals $(AgentName) @@ -216,29 +117,15 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' steps: - - script: | - hostnamectl - displayName: 'Host info' - workingDirectory: $(ROOT_DIR) # ------------------------------------------------------- - # Pull and build the code + # Build the code and create the containers. - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(Pipeline.Workspace) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) skip_azcli: "false" # ------------------------------------------------------- - template: 'azure-pipeline-templates/invalid-command-tests.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) + - ${{ if eq(parameters.exhaustive_test, true) }}: - template: 'azure-pipeline-templates/verbose-tests.yml' @@ -253,11 +140,7 @@ stages: client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) - container: $(containerName) config: $(BLOBFUSE2_CFG) - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) stress_dir: $(BLOBFUSE2_STRESS_DIR) huge_container: 'testcnt1' quick_stress: ${{ parameters.quick_stress }} @@ -270,15 +153,13 @@ stages: azurite_config: $(BLOBFUSE2_AZURITE_CFG) distro_name: $(AgentName) verbose_log: ${{ parameters.verbose_log }} - tags: $(tags) - template: azure-pipeline-templates/cleanup.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) + unmount: true + delete_containers: true - # Ubuntu Tests + # Ubuntu Tests for ADLS account - job: Set_2 timeoutInMinutes: 300 @@ -286,119 +167,25 @@ stages: matrix: Ubuntu-20-ADLS: AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-20' + poolName: 'blobfuse-ubuntu-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22-ADLS: AgentName: 'blobfuse-ubuntu22' - containerName: 'test-cnt-ubn-22' + poolName: 'blobfuse-ubuntu-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_SAS_CFG - value: '$(Pipeline.Workspace)/blobfuse2_sas_config.yaml' - - name: BLOBFUSE2_AZCLI_CFG - value: '$(Pipeline.Workspace)/blobfuse2_azcli_config.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: BLOBFUSE2_GTEST_CFG - value: '$(Pipeline.Workspace)/connection.yaml' - - name: BLOBFUSE2_AZURITE_CFG - value: '$(Pipeline.Workspace)/blobfuse2_azurite_config.yaml' - - name: BLOBFUSE2_STRESS_DIR - value: '$(Pipeline.Workspace)/blobfuse2_stress' - - name: DECODE_PERCENTS - value: false - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - working_directory: $(WORK_DIR) - root_dir: $(Pipeline.Workspace) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) - skip_ut: true # Skip UT because Block Blob set runs it - skip_azcli: "false" - - # ------------------------------------------------------- - - ${{ if eq(parameters.exhaustive_test, true) }}: - - template: 'azure-pipeline-templates/verbose-tests.yml' - parameters: - service: 'ADLS' - account_type: 'adls' - account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' - adls: true - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) - account_sas: $(adlsSas) - client_id: $(AZTEST_CLIENT) - tenant_id: $(AZTEST_TENANT) - client_secret: $(AZTEST_SECRET) - container: $(containerName) - config: $(BLOBFUSE2_ADLS_CFG) - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - stress_dir: $(BLOBFUSE2_STRESS_DIR) - huge_container: 'testcnt' - quick_stress: ${{ parameters.quick_stress }} - test_key_credential: true - test_sas_credential: true - test_azcli_credential: true - test_azurite: false - sas_credential_config: $(BLOBFUSE2_SAS_CFG) - azcli_credential_config: $(BLOBFUSE2_AZCLI_CFG) - azurite_config: $(BLOBFUSE2_AZURITE_CFG) - distro_name: $(AgentName) - verbose_log: ${{ parameters.verbose_log }} - tags: $(tags) - - - template: azure-pipeline-templates/cleanup.yml - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - - - job: Set_2_ARM - timeoutInMinutes: 300 - - strategy: - matrix: Ubuntu-22-ARM64-ADLS: AgentName: 'blobfuse-ubn22-arm64' - containerName: 'test-cnt-ubn-22-arm64' + poolName: 'blobfuse-ubn-arm64-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM) fuselib: 'libfuse3-dev' tags: 'fuse3' + pool: - name: "blobfuse-ubn-arm64-pool" + name: $(poolName) demands: - ImageOverride -equals $(AgentName) @@ -433,17 +220,9 @@ stages: steps: # ------------------------------------------------------- - # Pull and build the code + # Pull and build the code, Create the Containers in storage account - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(Pipeline.Workspace) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) skip_ut: true # Skip UT because Block Blob set runs it skip_azcli: "false" @@ -461,16 +240,12 @@ stages: client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) - container: $(containerName) config: $(BLOBFUSE2_ADLS_CFG) - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) stress_dir: $(BLOBFUSE2_STRESS_DIR) huge_container: 'testcnt' quick_stress: ${{ parameters.quick_stress }} test_key_credential: true - test_sas_credential: false + test_sas_credential: true test_azcli_credential: true test_azurite: false sas_credential_config: $(BLOBFUSE2_SAS_CFG) @@ -478,14 +253,12 @@ stages: azurite_config: $(BLOBFUSE2_AZURITE_CFG) distro_name: $(AgentName) verbose_log: ${{ parameters.verbose_log }} - tags: $(tags) - template: azure-pipeline-templates/cleanup.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - + unmount: true + delete_containers: true + - ${{ if eq(parameters.proxy_test, true) }}: # ----------------------------------------------------------- @@ -496,7 +269,6 @@ stages: matrix: Ubuntu-20-Proxy: AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-18-proxy' pool: name: "blobfuse-ubuntu-pool" @@ -523,40 +295,10 @@ stages: value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' steps: - - script: | - sudo systemctl stop apt-daily.timer - displayName: 'Stop apt update manager' - - # Start Proxy - - script: | - sudo apt-get update --fix-missing - sudo apt remove mitmproxy -y - sudo apt-get install python3-setuptools -y - sudo apt install python3-pip -y - sudo pip3 install werkzeug==2.3.7 - sudo pip3 install mitmproxy - mitmdump -q -w proxy_logs.txt & - displayName: 'Install & Start Proxy' - - # Configure Proxy cert & env - - script: | - sudo mkdir /usr/local/share/ca-certificates/extra - sudo cp ~/.mitmproxy/mitmproxy-ca-cert.cer /usr/local/share/ca-certificates/extra/mitmproxy-ca-cert.crt - sudo update-ca-certificates - export https_proxy="127.0.0.1:8080" - export http_proxy="127.0.0.1:8080" - displayName: 'Configure Proxy' - # ------------------------------------------------------- - # Pull, build and unit test the code + # Pull, build and unit test the code, create the containers. - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(Pipeline.Workspace) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) proxy_address: "127.0.0.1:8080" skip_ut: true @@ -664,33 +406,25 @@ stages: # ------------------------------------------------------------ - template: 'azure-pipeline-templates/cleanup.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) + unmount: true + delete_containers: true - script: | kill -9 $(pgrep mitmdump) displayName: 'Kill Proxy' - - - ${{ if eq(parameters.exhaustive_test, true) }}: - # RHEL Tests + - ${{ if eq(parameters.msi_test, true) }}: + # ----------------------------------------------------------- + # Ubuntu-20.04 MSI tests - job: Set_4 timeoutInMinutes: 60 strategy: matrix: - RHEL-8.6: - DistroVer: "RHEL-8.6" - Description: "Red Hat Enterprise Linux 8.6" - AgentName: "blobfuse-rhel8_6" - containerName: "test-cnt-rhel-86" - tags: 'fuse3' - RHEL-9.0: - DistroVer: "RHEL-9.0" - Description: "Red Hat Enterprise Linux 9.0" - AgentName: "blobfuse-rhel9" - containerName: "test-cnt-rhel-9" - tags: 'fuse3' + Ubuntu-20-MSI: + DistroVer: "Ubn20_MSI" + AgentName: "blobfuse-ubuntu20" + Description: "Ubuntu 20 MSI Test" + pool: - name: "blobfuse-rhel-pool" + name: "blobfuse-ubuntu-pool" demands: - ImageOverride -equals $(AgentName) @@ -701,9 +435,9 @@ stages: - name: TEMP_DIR value: '$(Pipeline.Workspace)/blobfuse2_tmp' - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' + value: '$(Pipeline.Workspace)/blobfuse2.msi.yaml' - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + value: '$(Pipeline.Workspace)/blobfuse2.msi.adls.yaml' - name: GOPATH value: '$(Pipeline.Workspace)/go' - name: ROOT_DIR @@ -714,304 +448,150 @@ stages: value: true steps: - - template: 'azure-pipeline-templates/distro-tests.yml' + # Print the agent info + - script: | + echo $(Description) + hostnamectl + displayName: 'Print Agent Info' + + # Build the code + - template: 'azure-pipeline-templates/build.yml' parameters: - working_dir: $(WORK_DIR) - root_dir: $(ROOT_DIR) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - config_path: $(BLOBFUSE2_CFG) - container: $(ContainerName) - blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - adls_account_name: $(AZTEST_ADLS_ACC_NAME) - adls_account_key: $(AZTEST_ADLS_KEY) + skip_msi: "false" + skip_azcli: "false" + + # BlockBlob MSI Test + - script: | + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) + displayName: "Create MSI Config File" + env: + NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) + NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) + ACCOUNT_TYPE: 'block' + ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' + VERBOSE_LOG: ${{ parameters.verbose_log }} + continueOnError: false + + - script: + cat $(BLOBFUSE2_CFG) + displayName: "Print config file" + + # Mount the container and run basic tests + - template: 'azure-pipeline-templates/verify-auth.yml' + parameters: + idstring: 'BlockBlob MSI' distro_name: $(AgentName) - gopath: $(GOPATH) - tags: $(tags) - installStep: + mountStep: script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo yum update -y - sudo yum groupinstall "Development Tools" -y - if [ $(AgentName) == "blobfuse-rhel9" ]; then - sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing - else - sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing - fi - displayName: 'Install fuse' - verbose_log: ${{ parameters.verbose_log }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) + displayName: 'AuthVerify MSI: Mount Container' + continueOnError: false - # Centos Tests - - job: Set_5 - timeoutInMinutes: 60 - strategy: - matrix: - CentOS-7.9: - DistroVer: "CentOS-7.9" - Description: "CentOS 7.9" - AgentName: "blobfuse-centos7" - ContainerName: "test-cnt-cent-7" - CentOS-8.5: - DistroVer: "CentOS-8.5" - Description: "CentOS 8.5" - AgentName: "blobfuse-centos8" - ContainerName: "test-cnt-cent-8" - - pool: - name: "blobfuse-centos-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: skipComponentGovernanceDetection - value: true - - steps: + # ADLS MSI Test - script: | - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - condition: eq(variables['AgentName'], 'blobfuse-centos8') - displayName: "Update OS mirrors" + cd $(WORK_DIR) + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) + displayName: "Create MSI Config File" + env: + NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) + ACCOUNT_TYPE: 'adls' + ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + VERBOSE_LOG: ${{ parameters.verbose_log }} + continueOnError: false - - template: 'azure-pipeline-templates/distro-tests.yml' + - script: + cat $(BLOBFUSE2_CFG) + displayName: "Print config file" + + # Mount the cntainer and run basic tests + - template: 'azure-pipeline-templates/verify-auth.yml' parameters: - working_dir: $(WORK_DIR) - root_dir: $(ROOT_DIR) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - config_path: $(BLOBFUSE2_CFG) - container: $(ContainerName) - blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - adls_account_name: $(AZTEST_ADLS_ACC_NAME) - adls_account_key: $(AZTEST_ADLS_KEY) + idstring: 'ADLS MSI' distro_name: $(AgentName) - gopath: $(GOPATH) - installStep: + mountStep: script: | - sudo yum update -y --skip-broken - if [ $(AgentName) == "blobfuse-centos8" ]; then - sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing - else - sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y - fi - displayName: 'Install fuse' - verbose_log: ${{ parameters.verbose_log }} + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) + displayName: 'AuthVerify MSI: Mount Container' + continueOnError: false - # Oracle Tests - - job: Set_6 + # Cleanup + - template: 'azure-pipeline-templates/cleanup.yml' + parameters: + unmount: true + delete_containers: true + + + - ${{ if eq(parameters.exhaustive_test, true) }}: + # Run E2E tests on different distros + - job: Set_5 + continueOnError: true timeoutInMinutes: 60 strategy: matrix: + RHEL-8.6: + DistroVer: 'RHEL-8.6' + distroId: '1' + poolName: 'blobfuse-rhel-pool' + Description: 'Red Hat Enterprise Linux 8.6' + AgentName: 'blobfuse-rhel8_6' + tags: 'fuse3' + RHEL-9.0: + DistroVer: 'RHEL-9.0' + distroId: '1' + poolName: 'blobfuse-rhel-pool' + Description: 'Red Hat Enterprise Linux 9.0' + AgentName: 'blobfuse-rhel9' + tags: 'fuse3' + CentOS-7.9: + DistroVer: "CentOS-7.9" + distroId: '2' + poolName: 'blobfuse-centos-pool' + Description: "CentOS 7.9" + AgentName: "blobfuse-centos7" + CentOS-8.5: + DistroVer: "CentOS-8.5" + distroId: '2' + poolName: 'blobfuse-centos-pool' + Description: "CentOS 8.5" + AgentName: "blobfuse-centos8" Oracle-8.1: DistroVer: "Oracle-8.1" + distroId: '3' + poolName: 'blobfuse-oracle-pool' Description: "Oracle Linux 8.1" AgentName: "blobfuse-oracle81" - ContainerName: "test-cnt-ora-81" - - pool: - name: "blobfuse-oracle-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: skipComponentGovernanceDetection - value: true - - steps: - - template: 'azure-pipeline-templates/distro-tests.yml' - parameters: - working_dir: $(WORK_DIR) - root_dir: $(ROOT_DIR) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - config_path: $(BLOBFUSE2_CFG) - container: $(ContainerName) - blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - adls_account_name: $(AZTEST_ADLS_ACC_NAME) - adls_account_key: $(AZTEST_ADLS_KEY) - distro_name: $(AgentName) - gopath: $(GOPATH) - installStep: - script: | - sudo yum update -y - sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing - displayName: 'Install fuse' - verbose_log: ${{ parameters.verbose_log }} - - # ROCKY Tests - - job: Set_7 - timeoutInMinutes: 60 - strategy: - matrix: Rocky-8.0: DistroVer: "Rocky-8.0" + distroId: '4' + poolName: 'blobfuse2-rocky-pool' Description: "Rocky Linux 8.0" AgentName: "blobfuse-rocky8" - ContainerName: "test-cnt-rocky-8" tags: 'fuse3' Rocky-9.0: DistroVer: "Rocky-9.0" + distroId: '4' + poolName: 'blobfuse2-rocky-pool' Description: "Rocky Linux 9.0" AgentName: "blobfuse-rocky9" - ContainerName: "test-cnt-rocky-9" tags: 'fuse3' - pool: - name: "blobfuse2-rocky-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: skipComponentGovernanceDetection - value: true - - steps: - - template: 'azure-pipeline-templates/distro-tests.yml' - parameters: - working_dir: $(WORK_DIR) - root_dir: $(ROOT_DIR) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - config_path: $(BLOBFUSE2_CFG) - container: $(ContainerName) - blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - adls_account_name: $(AZTEST_ADLS_ACC_NAME) - adls_account_key: $(AZTEST_ADLS_KEY) - distro_name: $(AgentName) - gopath: $(GOPATH) - tags: $(tags) - installStep: - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo yum update -y - sudo yum groupinstall "Development Tools" -y - sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing - displayName: 'Install fuse' - verbose_log: ${{ parameters.verbose_log }} - - # # SUSE Tests - # - job: Set_9 - # timeoutInMinutes: 60 - # strategy: - # matrix: - # SUSE-15: - # DistroVer: "SUSE-15" - # Description: "SUSE Enterprise Linux 15" - # AgentName: "blobfuse-suse15" - # ContainerName: "test-cnt-suse-15" - - # pool: - # name: "blobfuse-suse-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - # - name: MOUNT_DIR - # value: '$(Pipeline.Workspace)/blob_mnt' - # - name: TEMP_DIR - # value: '$(Pipeline.Workspace)/blobfuse2_tmp' - # - name: BLOBFUSE2_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.yaml' - # - name: BLOBFUSE2_ADLS_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - # - name: GOPATH - # value: '$(Pipeline.Workspace)/go' - # - name: ROOT_DIR - # value: '$(System.DefaultWorkingDirectory)' - # - name: WORK_DIR - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - # - name: skipComponentGovernanceDetection - # value: true - - # steps: - # - template: 'azure-pipeline-templates/distro-tests.yml' - # parameters: - # working_dir: $(WORK_DIR) - # root_dir: $(ROOT_DIR) - # temp_dir: $(TEMP_DIR) - # mount_dir: $(MOUNT_DIR) - # config_path: $(BLOBFUSE2_CFG) - # container: $(ContainerName) - # blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - # blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - # adls_account_name: $(AZTEST_ADLS_ACC_NAME) - # adls_account_key: $(AZTEST_ADLS_KEY) - # distro_name: $(AgentName) - # gopath: $(GOPATH) - # installStep: - # script: | - # sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse - # wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm - # sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm - # fusermount3 --v - # wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - # sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - # displayName: 'Install fuse' - # verbose_log: ${{ parameters.verbose_log }} - - # Mariner Tests - - job: Set_10 - timeoutInMinutes: 60 - strategy: - matrix: + SUSE-15: + DistroVer: "SUSE-15" + distroId: '5' + poolName: 'blobfuse-suse-pool' + Description: "SUSE Enterprise Linux 15" + AgentName: "blobfuse-suse15" Mariner2: DistroVer: "Mariner2" + distroId: '6' + poolName: 'blobfuse-mariner-pool' Description: "CBL-Mariner2 Linux" AgentName: "blobfuse-mariner2" - ContainerName: "test-cnt-mari-2" fuselib: 'libfuse3-dev' - tags: 'fuse3' + tags: 'fuse3' pool: - name: "blobfuse-mariner-pool" + name: $(poolName) demands: - ImageOverride -equals $(AgentName) @@ -1048,150 +628,10 @@ stages: adls_account_name: $(AZTEST_ADLS_ACC_NAME) adls_account_key: $(AZTEST_ADLS_KEY) distro_name: $(AgentName) - tags: $(tags) - fuselib: $(fuselib) gopath: $(GOPATH) - installStep: - script: | - sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y - displayName: 'Install fuse' + tags: $(tags) verbose_log: ${{ parameters.verbose_log }} - - ${{ if eq(parameters.msi_test, true) }}: - # ----------------------------------------------------------- - # Ubuntu-20.04 MSI tests - - job: Set_11 - timeoutInMinutes: 60 - strategy: - matrix: - Ubuntu-20-MSI: - DistroVer: "Ubn20_MSI" - AgentName: "blobfuse-ubuntu20" - Description: "Ubuntu 20 MSI Test" - ContainerName: "test-cnt-ubn-20-msi" - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.msi.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.msi.adls.yaml' - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - name: skipComponentGovernanceDetection - value: true - - steps: - # Print the agent info - - script: | - echo $(Description) - hostnamectl - displayName: 'Print Agent Info' - - # Create directory structure - - script: | - sudo rm -rf $(ROOT_DIR) - sudo mkdir -p $(ROOT_DIR) - sudo chown -R `whoami` $(ROOT_DIR) - chmod 777 $(ROOT_DIR) - mkdir -p $(ROOT_DIR)/go/src - displayName: 'Create Directory Structure' - - # Build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(ContainerName) - skip_msi: "false" - skip_azcli: "false" - - # BlockBlob MSI Test - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) - displayName: "Create MSI Config File" - env: - NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) - NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' - VERBOSE_LOG: ${{ parameters.verbose_log }} - continueOnError: false - - - script: - cat $(BLOBFUSE2_CFG) - displayName: "Print config file" - - # Mount the cntainer and run basic tests - - template: 'azure-pipeline-templates/verify-auth.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - idstring: 'BlockBlob MSI' - distro_name: $(AgentName) - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) - displayName: 'AuthVerify MSI: Mount Container' - continueOnError: false - - # ADLS MSI Test - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) - displayName: "Create MSI Config File" - env: - NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' - VERBOSE_LOG: ${{ parameters.verbose_log }} - continueOnError: false - - - script: - cat $(BLOBFUSE2_CFG) - displayName: "Print config file" - - # Mount the cntainer and run basic tests - - template: 'azure-pipeline-templates/verify-auth.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - idstring: 'ADLS MSI' - distro_name: $(AgentName) - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=$(BLOBFUSE2_CFG) - displayName: 'AuthVerify MSI: Mount Container' - continueOnError: false - - # Cleanup - - template: 'azure-pipeline-templates/cleanup.yml' - parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - - - stage: BlockCacheDataValidation jobs: # Ubuntu Tests From e828b973546298ee9db04dd322480f048a223624 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 13:49:00 +0000 Subject: [PATCH 26/87] Cleanup nytly pipeline --- blobfuse2-nightly.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index de664ce2d..6eb17fd03 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -66,13 +66,11 @@ stages: Ubuntu-20-BlockBlob: AgentName: 'blobfuse-ubuntu20' poolName: 'blobfuse-ubuntu-pool' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22-BlockBlob: AgentName: 'blobfuse-ubuntu22' poolName: 'blobfuse-ubuntu-pool' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' Ubuntu-22-ARM64-BlockBlob: From 4dc09a2b61db4cb9c6ad7586e34d52d50791ed56 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 13:53:24 +0000 Subject: [PATCH 27/87] Change parameter for proxy setup --- azure-pipeline-templates/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 2b6458d14..c9f4388de 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -23,7 +23,7 @@ parameters: default: true - name: proxy_address type: string - default: "" + default: "no_proxy" - name: skip_ut type: boolean default: false @@ -44,7 +44,7 @@ steps: # If proxy is given, Install and setup the proxy, these tests can only be done on ubuntu - - ${{ if ne(parameters.proxy_address, "") }}: + - ${{ if ne(parameters.proxy_address, "no_proxy") }}: # Start Proxy - script: | sudo apt-get update --fix-missing From 1504fea86edec2af7e70d840b16ec85b137cfa51 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 13:57:53 +0000 Subject: [PATCH 28/87] Change parameter for proxy setup --- azure-pipeline-templates/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index c9f4388de..5ed844023 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -23,7 +23,7 @@ parameters: default: true - name: proxy_address type: string - default: "no_proxy" + default: 'no_proxy' - name: skip_ut type: boolean default: false @@ -44,7 +44,7 @@ steps: # If proxy is given, Install and setup the proxy, these tests can only be done on ubuntu - - ${{ if ne(parameters.proxy_address, "no_proxy") }}: + - ${{ if ne(parameters.proxy_address, 'no_proxy') }}: # Start Proxy - script: | sudo apt-get update --fix-missing From 9067298e9b3256e6abb5e6e85d3d93ba12ffe93c Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 13:59:31 +0000 Subject: [PATCH 29/87] typo --- azure-pipeline-templates/package-install.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 022b323ad..39a8ca55a 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -18,6 +18,7 @@ parameters: # UBUNTU PACKAGE INSTALLATION # Before installation, stop any process such as update manager from using package manager. # otherwise installation may fail +steps: - script: | sleep 30 sudo systemctl stop apt-daily.timer From 7d88a0f7a1c639003e981410dd2e2f75eda47ae0 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 14:33:25 +0000 Subject: [PATCH 30/87] Change package installation --- azure-pipeline-templates/package-install.yml | 154 +++++++++---------- 1 file changed, 69 insertions(+), 85 deletions(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 39a8ca55a..4b23c89c3 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -20,91 +20,75 @@ parameters: # otherwise installation may fail steps: - script: | - sleep 30 - sudo systemctl stop apt-daily.timer - echo "Add Lock Timeout for apt package manager" - sudo sh -c 'echo "DPkg::Lock::Timeout \"120\";" > /etc/apt/apt.conf.d/99timeout' - sudo ps -aux | grep -iE "apt" - sudo killall apt apt-get - yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock - echo "Released any lock if some other process has acquired" - sudo dpkg --configure -a +# ubuntu + if [ "$distro_id" == "0" ]; then + sleep 30 + sudo systemctl stop apt-daily.timer + echo "Add Lock Timeout for apt package manager" + sudo sh -c 'echo "DPkg::Lock::Timeout \"120\";" > /etc/apt/apt.conf.d/99timeout' + sudo ps -aux | grep -iE "apt" + sudo killall apt apt-get + yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock + echo "Released any lock if some other process has acquired" + sudo dpkg --configure -a - echo "****************Starting Updates and Installation of Packages**************" - sudo apt-get update --fix-missing - sudo apt-get install make cmake gcc g++ git parallel -y - if [ "$(tags)" == "fuse2" ] - then - sudo apt-get install libfuse2 libfuse-dev -y - else - sudo apt-get install fuse3 libfuse3-dev -y + echo "****************Starting Updates and Installation of Packages**************" + sudo apt-get update --fix-missing + sudo apt-get install make cmake gcc g++ git parallel -y + if [ "$(tags)" == "fuse2" ] + then + sudo apt-get install libfuse2 libfuse-dev -y + else + sudo apt-get install fuse3 libfuse3-dev -y + fi + fusermount3 -V + fusermount -V + echo "***********************Install AzCli*************************" + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version +# rhel + elif [ "$distro_id" == "1" ]; then + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo yum update -y + sudo yum groupinstall "Development Tools" -y + if [ $(AgentName) == "blobfuse-rhel9" ]; then + sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing + else + sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing + fi +# centos + elif [ "$distro_id" == "2" ]; then + if [ "$(AgentName)" == "blobfuse-centos8" ] + then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + sudo yum update -y --skip-broken + if [ $(AgentName) == "blobfuse-centos8" ]; then + sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing + else + sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y + fi +# oracle + elif [ "$distro_id" == "3" ]; then + sudo yum update -y + sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing +# rocky + elif [ "$distro_id" == "4" ]; then + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo yum update -y + sudo yum groupinstall "Development Tools" -y + sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing +# suse + elif [ "$distro_id" == "5" ]; then + sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse + wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm + sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm + fusermount3 --v + wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm + sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm +# mariner + elif [ "$distro_id" == "6" ]; then + sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y fi - fusermount3 -V - fusermount -V - echo "***********************Install AzCli*************************" - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - az --version - condition: eq('${{ parameters.distro_id }}', "0") - displayName: 'Libfuse Setup ${{ parameters.distro_name }}' - - # RHEL PACKAGE INSTALLATION - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo yum update -y - sudo yum groupinstall "Development Tools" -y - if [ $(AgentName) == "blobfuse-rhel9" ]; then - sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing - else - sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing - fi - condition: eq('${{ parameters.distro_id }}', "1") - displayName: 'Libfuse Setup ${{ parameters.distro_name }}' - - # CENTOS PACKAGE INSTALLATION - - script: | - if [ "$(AgentName)" == "blobfuse-centos8" ] - then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - sudo yum update -y --skip-broken - if [ $(AgentName) == "blobfuse-centos8" ]; then - sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing - else - sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y - fi - condition: eq('${{ parameters.distro_id }}', "2") - displayName: 'Libfuse Setup ${{ parameters.distro_name }}' - - # ORACLE PACKAGE INSTALLATION - - script: | - sudo yum update -y - sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing - condition: eq('${{ parameters.distro_id }}', "3") - displayName: 'Libfuse Setup ${{ parameters.distro_name }}' - - # ROCKY PACKAGE INSTALLATION - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo yum update -y - sudo yum groupinstall "Development Tools" -y - sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing - condition: eq('${{ parameters.distro_id }}', "4") - displayName: 'Libfuse Setup ${{ parameters.distro_name }}' - - # SUSE PACKAGE INSTALLATION - - script: | - sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse - wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm - fusermount3 --v - wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - condition: eq('${{ parameters.distro_id }}', "5") - displayName: 'Libfuse Setup ${{ parameters.distro_name }}' - - # MARINER PACKAGE INSTALLATION - - script: | - sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y - condition: eq('${{ parameters.distro_id }}', "6") displayName: 'Libfuse Setup ${{ parameters.distro_name }}' From abd3c0fbcc2e1848148e4e279c7ffa9b970c6ad0 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 14:42:03 +0000 Subject: [PATCH 31/87] Change package installation --- azure-pipeline-templates/package-install.yml | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 4b23c89c3..8ee24e2bd 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -21,7 +21,7 @@ parameters: steps: - script: | # ubuntu - if [ "$distro_id" == "0" ]; then + if [ "$(distro_id)" == "0" ]; then sleep 30 sudo systemctl stop apt-daily.timer echo "Add Lock Timeout for apt package manager" @@ -47,7 +47,7 @@ steps: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash az --version # rhel - elif [ "$distro_id" == "1" ]; then + elif [ "$(distro_id)" == "1" ]; then sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo yum update -y sudo yum groupinstall "Development Tools" -y @@ -57,7 +57,7 @@ steps: sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing fi # centos - elif [ "$distro_id" == "2" ]; then + elif [ "$(distro_id)" == "2" ]; then if [ "$(AgentName)" == "blobfuse-centos8" ] then sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* @@ -70,17 +70,17 @@ steps: sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y fi # oracle - elif [ "$distro_id" == "3" ]; then + elif [ "$(distro_id)" == "3" ]; then sudo yum update -y sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing # rocky - elif [ "$distro_id" == "4" ]; then + elif [ "$(distro_id)" == "4" ]; then sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo yum update -y sudo yum groupinstall "Development Tools" -y sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing # suse - elif [ "$distro_id" == "5" ]; then + elif [ "$(distro_id)" == "5" ]; then sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm @@ -88,7 +88,9 @@ steps: wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm # mariner - elif [ "$distro_id" == "6" ]; then + elif [ "$(distro_id)" == "6" ]; then sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y fi displayName: 'Libfuse Setup ${{ parameters.distro_name }}' + env: + distro_id: ${{ parameters.distro_id }} From 324f6eee476c8497d6fbf5e14c452461964c8535 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 14:47:30 +0000 Subject: [PATCH 32/87] Change package installation --- azure-pipeline-templates/package-install.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 8ee24e2bd..c4428192b 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -46,8 +46,8 @@ steps: echo "***********************Install AzCli*************************" curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash az --version -# rhel elif [ "$(distro_id)" == "1" ]; then + # rhel sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo yum update -y sudo yum groupinstall "Development Tools" -y @@ -56,8 +56,8 @@ steps: else sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing fi -# centos elif [ "$(distro_id)" == "2" ]; then + # centos if [ "$(AgentName)" == "blobfuse-centos8" ] then sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* @@ -69,26 +69,26 @@ steps: else sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y fi -# oracle elif [ "$(distro_id)" == "3" ]; then + # oracle sudo yum update -y sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing -# rocky elif [ "$(distro_id)" == "4" ]; then + # rocky sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo yum update -y sudo yum groupinstall "Development Tools" -y sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing -# suse elif [ "$(distro_id)" == "5" ]; then + # suse sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm fusermount3 --v wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm -# mariner elif [ "$(distro_id)" == "6" ]; then + # mariner sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y fi displayName: 'Libfuse Setup ${{ parameters.distro_name }}' From e6000c1f87aefb2a790518d87cb2d955e86b8879 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Fri, 14 Feb 2025 14:51:14 +0000 Subject: [PATCH 33/87] Change package installation --- azure-pipeline-templates/package-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index c4428192b..39afab545 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -20,7 +20,7 @@ parameters: # otherwise installation may fail steps: - script: | -# ubuntu + # ubuntu if [ "$(distro_id)" == "0" ]; then sleep 30 sudo systemctl stop apt-daily.timer From 5e10d68cd92edc7cbf73f119d17da555ac176acd Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 05:53:40 +0000 Subject: [PATCH 34/87] Modify package install --- azure-pipeline-templates/build.yml | 10 +-- .../generate-container-name.yml | 1 - azure-pipeline-templates/package-install.yml | 75 ++++++++++--------- blobfuse2-ci.yaml | 6 ++ blobfuse2-nightly.yaml | 18 ++--- 5 files changed, 56 insertions(+), 54 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 5ed844023..922a3ef10 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -6,12 +6,6 @@ # 4. Run Unit Tests if specified by parameter skip_ut: false parameters: - - name: distro_name - type: string - default: "UBUNTU" - - name: distro_id - type: string - default: "0" - name: skip_msi type: string default: "true" @@ -39,8 +33,8 @@ steps: # Package manager installs for libfuse, - template: 'package-install.yml' parameters: - distro_name: $(paramters.distro_name) - distro_id: $(paramters.distro_id) + distro: $(distro) + distro_version: $(distro_version) # If proxy is given, Install and setup the proxy, these tests can only be done on ubuntu diff --git a/azure-pipeline-templates/generate-container-name.yml b/azure-pipeline-templates/generate-container-name.yml index d16680e61..79a6bc15b 100644 --- a/azure-pipeline-templates/generate-container-name.yml +++ b/azure-pipeline-templates/generate-container-name.yml @@ -3,5 +3,4 @@ steps: CONTAINER_NAME=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) echo "Generated container name: $CONTAINER_NAME" echo "##vso[task.setvariable variable=containerName]$CONTAINER_NAME" - echo "Assigned container name variable: $(containerName)" displayName: 'Generate random container name' diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 39afab545..af2926987 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -1,27 +1,22 @@ -# If you want to add new disto, create the next available distro_id -# Currently, Supported distros, distro Id -# UBUNTU 0 -# RHEL 1 -# CENTOS 2 -# ORACLE 3 -# ROCKY 4 -# SUSE 5 -# MARINER 6 parameters: - - name: distro_name + - name: distro type: string - default: "UBUNTU" - - name: distro_id + default: ubuntu + values: + - ubuntu + - rhel + - centos + - oracle + - rocky + - suse + - mariner + - name: distro_version type: string - default: "0" + default: ubuntu - # UBUNTU PACKAGE INSTALLATION - # Before installation, stop any process such as update manager from using package manager. - # otherwise installation may fail steps: - - script: | - # ubuntu - if [ "$(distro_id)" == "0" ]; then + - ${{ if eq(parameters.distro, 'ubuntu') }}: + - script: | sleep 30 sudo systemctl stop apt-daily.timer echo "Add Lock Timeout for apt package manager" @@ -31,7 +26,6 @@ steps: yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock echo "Released any lock if some other process has acquired" sudo dpkg --configure -a - echo "****************Starting Updates and Installation of Packages**************" sudo apt-get update --fix-missing sudo apt-get install make cmake gcc g++ git parallel -y @@ -46,8 +40,10 @@ steps: echo "***********************Install AzCli*************************" curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash az --version - elif [ "$(distro_id)" == "1" ]; then - # rhel + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + # RHEL PACKAGE INSTALLATION + - ${{ if eq(parameters.distro, 'rhel') }}: + - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo yum update -y sudo yum groupinstall "Development Tools" -y @@ -56,8 +52,10 @@ steps: else sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing fi - elif [ "$(distro_id)" == "2" ]; then - # centos + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + # CENTOS PACKAGE INSTALLATION + - ${{ if eq(parameters.distro, 'centos') }}: + - script: | if [ "$(AgentName)" == "blobfuse-centos8" ] then sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* @@ -69,28 +67,33 @@ steps: else sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y fi - elif [ "$(distro_id)" == "3" ]; then - # oracle + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + # ORACLE PACKAGE INSTALLATION + - ${{ if eq(parameters.distro, 'oracle') }}: + - script: | sudo yum update -y sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing - elif [ "$(distro_id)" == "4" ]; then - # rocky + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + # ROCKY PACKAGE INSTALLATION + - ${{ if eq(parameters.distro, 'rocky') }}: + - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo sudo yum update -y sudo yum groupinstall "Development Tools" -y sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing - elif [ "$(distro_id)" == "5" ]; then - # suse + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + # SUSE PACKAGE INSTALLATION + - ${{ if eq(parameters.distro, 'suse') }}: + - script: | sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm fusermount3 --v wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - elif [ "$(distro_id)" == "6" ]; then - # mariner + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + # MARINER PACKAGE INSTALLATION + - ${{ if eq(parameters.distro, 'mariner') }}: + - script: | sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y - fi - displayName: 'Libfuse Setup ${{ parameters.distro_name }}' - env: - distro_id: ${{ parameters.distro_id }} + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' \ No newline at end of file diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 59338c35c..eadfa9b8c 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -20,18 +20,24 @@ jobs: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' + distro: ubuntu + distro_version: ubuntu_20 poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse-dev' tags: 'fuse2' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) Ubuntu-22: AgentName: 'blobfuse-ubuntu22' + distro: ubuntu + distro_version: ubuntu_22_x86 poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse3-dev' tags: 'fuse3' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) Ubuntu-22-ARM64: AgentName: 'blobfuse-ubn22-arm64' + distro: ubuntu + distro_version: ubuntu_22_arm poolName: 'blobfuse-ubn-arm64-pool' fuselib: 'libfuse3-dev' tags: 'fuse3' diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 6eb17fd03..2988c8427 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -530,59 +530,59 @@ stages: matrix: RHEL-8.6: DistroVer: 'RHEL-8.6' - distroId: '1' + distro: rhel poolName: 'blobfuse-rhel-pool' Description: 'Red Hat Enterprise Linux 8.6' AgentName: 'blobfuse-rhel8_6' tags: 'fuse3' RHEL-9.0: DistroVer: 'RHEL-9.0' - distroId: '1' + distro: rhel poolName: 'blobfuse-rhel-pool' Description: 'Red Hat Enterprise Linux 9.0' AgentName: 'blobfuse-rhel9' tags: 'fuse3' CentOS-7.9: DistroVer: "CentOS-7.9" - distroId: '2' + distro: centos poolName: 'blobfuse-centos-pool' Description: "CentOS 7.9" AgentName: "blobfuse-centos7" CentOS-8.5: DistroVer: "CentOS-8.5" - distroId: '2' + distro: centos poolName: 'blobfuse-centos-pool' Description: "CentOS 8.5" AgentName: "blobfuse-centos8" Oracle-8.1: DistroVer: "Oracle-8.1" - distroId: '3' + distro: oracle poolName: 'blobfuse-oracle-pool' Description: "Oracle Linux 8.1" AgentName: "blobfuse-oracle81" Rocky-8.0: DistroVer: "Rocky-8.0" - distroId: '4' + distro: rocky poolName: 'blobfuse2-rocky-pool' Description: "Rocky Linux 8.0" AgentName: "blobfuse-rocky8" tags: 'fuse3' Rocky-9.0: DistroVer: "Rocky-9.0" - distroId: '4' + distro: rocky poolName: 'blobfuse2-rocky-pool' Description: "Rocky Linux 9.0" AgentName: "blobfuse-rocky9" tags: 'fuse3' SUSE-15: DistroVer: "SUSE-15" - distroId: '5' + distro: suse poolName: 'blobfuse-suse-pool' Description: "SUSE Enterprise Linux 15" AgentName: "blobfuse-suse15" Mariner2: DistroVer: "Mariner2" - distroId: '6' + distro: mariner poolName: 'blobfuse-mariner-pool' Description: "CBL-Mariner2 Linux" AgentName: "blobfuse-mariner2" From f0e93679aa7532470b1579cc0aa9b4ff7fe1dcc6 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 05:57:08 +0000 Subject: [PATCH 35/87] Modify package install --- blobfuse2-ci.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index eadfa9b8c..5c69895d8 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -20,24 +20,24 @@ jobs: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' - distro: ubuntu - distro_version: ubuntu_20 + distro: 'ubuntu' + distro_version: 'ubuntu_20' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse-dev' tags: 'fuse2' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) Ubuntu-22: AgentName: 'blobfuse-ubuntu22' - distro: ubuntu - distro_version: ubuntu_22_x86 + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse3-dev' tags: 'fuse3' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) Ubuntu-22-ARM64: AgentName: 'blobfuse-ubn22-arm64' - distro: ubuntu - distro_version: ubuntu_22_arm + distro: 'ubuntu' + distro_version: 'ubuntu_22_arm' poolName: 'blobfuse-ubn-arm64-pool' fuselib: 'libfuse3-dev' tags: 'fuse3' From 7d03920f36df80794a37171dda3e3f01282f83d1 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 06:54:07 +0000 Subject: [PATCH 36/87] Modify package install --- azure-pipeline-templates/package-install.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index af2926987..b72a5ca28 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -2,14 +2,14 @@ parameters: - name: distro type: string default: ubuntu - values: - - ubuntu - - rhel - - centos - - oracle - - rocky - - suse - - mariner + # values: + # - ubuntu + # - rhel + # - centos + # - oracle + # - rocky + # - suse + # - mariner - name: distro_version type: string default: ubuntu From 1c5bcd2ad04a6370ddf58e10a0edfb565ff514f8 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 07:04:08 +0000 Subject: [PATCH 37/87] Modify package install --- azure-pipeline-templates/package-install.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index b72a5ca28..424bb4204 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -15,6 +15,9 @@ parameters: default: ubuntu steps: + - scripe: | + echo ${{ parameters.distro }} + displayName: 'Check on something' - ${{ if eq(parameters.distro, 'ubuntu') }}: - script: | sleep 30 From c3044f5f3ec1f8dfc447a761df4062c4c18b35f7 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 07:05:10 +0000 Subject: [PATCH 38/87] Modify package install --- azure-pipeline-templates/package-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 424bb4204..b9ab0e2bd 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -15,7 +15,7 @@ parameters: default: ubuntu steps: - - scripe: | + - script: | echo ${{ parameters.distro }} displayName: 'Check on something' - ${{ if eq(parameters.distro, 'ubuntu') }}: From 80e44cdbc02319271075199284b7faaea69b6a5d Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 07:28:29 +0000 Subject: [PATCH 39/87] Modify package install --- azure-pipeline-templates/package-install.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index b9ab0e2bd..da34abfba 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -44,7 +44,8 @@ steps: curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash az --version displayName: 'Libfuse Setup ${{ parameters.distro_version }}' - # RHEL PACKAGE INSTALLATION + + # RHEL PACKAGE INSTALLATION - ${{ if eq(parameters.distro, 'rhel') }}: - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo @@ -56,7 +57,8 @@ steps: sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing fi displayName: 'Libfuse Setup ${{ parameters.distro_version }}' - # CENTOS PACKAGE INSTALLATION + + # CENTOS PACKAGE INSTALLATION - ${{ if eq(parameters.distro, 'centos') }}: - script: | if [ "$(AgentName)" == "blobfuse-centos8" ] @@ -71,13 +73,15 @@ steps: sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y fi displayName: 'Libfuse Setup ${{ parameters.distro_version }}' - # ORACLE PACKAGE INSTALLATION + + # ORACLE PACKAGE INSTALLATION - ${{ if eq(parameters.distro, 'oracle') }}: - script: | sudo yum update -y sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing displayName: 'Libfuse Setup ${{ parameters.distro_version }}' - # ROCKY PACKAGE INSTALLATION + + # ROCKY PACKAGE INSTALLATION - ${{ if eq(parameters.distro, 'rocky') }}: - script: | sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo @@ -85,7 +89,8 @@ steps: sudo yum groupinstall "Development Tools" -y sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing displayName: 'Libfuse Setup ${{ parameters.distro_version }}' - # SUSE PACKAGE INSTALLATION + + # SUSE PACKAGE INSTALLATION - ${{ if eq(parameters.distro, 'suse') }}: - script: | sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse @@ -95,7 +100,8 @@ steps: wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm displayName: 'Libfuse Setup ${{ parameters.distro_version }}' - # MARINER PACKAGE INSTALLATION + + # MARINER PACKAGE INSTALLATION - ${{ if eq(parameters.distro, 'mariner') }}: - script: | sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y From da9126445ab5c71f66ea185460ecd5d058b26728 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 07:37:53 +0000 Subject: [PATCH 40/87] Modify package install --- azure-pipeline-templates/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 922a3ef10..34c0abdf9 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -33,7 +33,7 @@ steps: # Package manager installs for libfuse, - template: 'package-install.yml' parameters: - distro: $(distro) + distro: ubuntu distro_version: $(distro_version) From 0c1106432f6152072f81a96a70f460b71cdb2560 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 08:11:57 +0000 Subject: [PATCH 41/87] Modify package install --- azure-pipeline-templates/build.yml | 2 +- azure-pipeline-templates/container.yml | 34 +++++++++++++------------- blobfuse2-ci.yaml | 6 ++--- 3 files changed, 21 insertions(+), 21 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 34c0abdf9..922a3ef10 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -33,7 +33,7 @@ steps: # Package manager installs for libfuse, - template: 'package-install.yml' parameters: - distro: ubuntu + distro: $(distro) distro_version: $(distro_version) diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index 43e98647c..fef9b8a64 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -18,22 +18,22 @@ parameters: steps: # Creating the Container - - script: | - az --version - echo "Creating Container: ${{ parameters.container_name }}" - az storage container create --name ${{ parameters.container_name }} \ - --account-name ${{ parameters.account_name }} \ - --account-key ${{ parameters.account_key }} \ - --fail-on-exist - displayName: 'Creating ${{ parameters.account_type }} Container' - condition: eq('${{ parameters.create_container }}', true) + - ${{ eq(parameters.create_container, true) }}: + - script: | + az --version + echo "Creating Container: ${{ parameters.container_name }}" + az storage container create --name ${{ parameters.container_name }} \ + --account-name ${{ parameters.account_name }} \ + --account-key ${{ parameters.account_key }} \ + --fail-on-exist + displayName: 'Creating ${{ parameters.account_type }} Container' # Destroying the Container - - script: | - az --version - echo "Deleting Container: ${{ parameters.container_name }}" - az storage container delete --name ${{ parameters.container_name }} \ - --account-name ${{ parameters.account_name }} \ - --account-key ${{ parameters.account_key }} \ - displayName: 'Deleting ${{ parameters.account_type }} Container' - condition: eq('${{ parameters.delete_container }}', true) \ No newline at end of file + - ${{ eq(parameters.delete_container, true) }}: + - script: | + az --version + echo "Deleting Container: ${{ parameters.container_name }}" + az storage container delete --name ${{ parameters.container_name }} \ + --account-name ${{ parameters.account_name }} \ + --account-key ${{ parameters.account_key }} \ + displayName: 'Deleting ${{ parameters.account_type }} Container' \ No newline at end of file diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 5c69895d8..1eb5018fa 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -20,7 +20,7 @@ jobs: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' - distro: 'ubuntu' + distro: ubuntu distro_version: 'ubuntu_20' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse-dev' @@ -28,7 +28,7 @@ jobs: adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) Ubuntu-22: AgentName: 'blobfuse-ubuntu22' - distro: 'ubuntu' + distro: ubuntu distro_version: 'ubuntu_22_x86' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse3-dev' @@ -36,7 +36,7 @@ jobs: adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) Ubuntu-22-ARM64: AgentName: 'blobfuse-ubn22-arm64' - distro: 'ubuntu' + distro: ubuntu distro_version: 'ubuntu_22_arm' poolName: 'blobfuse-ubn-arm64-pool' fuselib: 'libfuse3-dev' From bc055c9bbef9e50ac4f7218d892543a5f74b752f Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 08:17:15 +0000 Subject: [PATCH 42/87] Modify package install --- azure-pipeline-templates/container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index fef9b8a64..b7c0931d1 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -18,7 +18,7 @@ parameters: steps: # Creating the Container - - ${{ eq(parameters.create_container, true) }}: + - ${{ if eq(parameters.create_container, true) }}: - script: | az --version echo "Creating Container: ${{ parameters.container_name }}" @@ -29,7 +29,7 @@ steps: displayName: 'Creating ${{ parameters.account_type }} Container' # Destroying the Container - - ${{ eq(parameters.delete_container, true) }}: + - ${{ if eq(parameters.delete_container, true) }}: - script: | az --version echo "Deleting Container: ${{ parameters.container_name }}" From 8195e21dd70742ce4370b9bb4d3bb1ac02ce0343 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 08:31:13 +0000 Subject: [PATCH 43/87] Modify package install --- azure-pipeline-templates/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 922a3ef10..bb25ffb39 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -33,8 +33,8 @@ steps: # Package manager installs for libfuse, - template: 'package-install.yml' parameters: - distro: $(distro) - distro_version: $(distro_version) + # distro: $(distro) + # distro_version: $(distro_version) # If proxy is given, Install and setup the proxy, these tests can only be done on ubuntu From 540c91ceb0c3c4d264b79f98fb3256e2ea103ab1 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 08:32:19 +0000 Subject: [PATCH 44/87] Modify package install --- azure-pipeline-templates/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index bb25ffb39..622a19034 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -32,7 +32,7 @@ steps: # Package manager installs for libfuse, - template: 'package-install.yml' - parameters: + #parameters: # distro: $(distro) # distro_version: $(distro_version) From 03defa6ac0d37b444d162d23d475537bb9079080 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 16:51:15 +0000 Subject: [PATCH 45/87] Modify package install --- azure-pipeline-templates/build.yml | 6 +++--- azure-pipeline-templates/package-install.yml | 5 +++++ blobfuse2-ci.yaml | 6 +++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 622a19034..922a3ef10 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -32,9 +32,9 @@ steps: # Package manager installs for libfuse, - template: 'package-install.yml' - #parameters: - # distro: $(distro) - # distro_version: $(distro_version) + parameters: + distro: $(distro) + distro_version: $(distro_version) # If proxy is given, Install and setup the proxy, these tests can only be done on ubuntu diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index da34abfba..d16a05a57 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -18,7 +18,12 @@ steps: - script: | echo ${{ parameters.distro }} displayName: 'Check on something' + - ${{ if eq(parameters.distro, 'ubuntu') }}: + - script: | + echo "these are equal" + displayName: "these are equal" + - ${{ if ne(parameters.distro, 'ubuntu') }}: - script: | sleep 30 sudo systemctl stop apt-daily.timer diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 1eb5018fa..5c69895d8 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -20,7 +20,7 @@ jobs: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' - distro: ubuntu + distro: 'ubuntu' distro_version: 'ubuntu_20' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse-dev' @@ -28,7 +28,7 @@ jobs: adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) Ubuntu-22: AgentName: 'blobfuse-ubuntu22' - distro: ubuntu + distro: 'ubuntu' distro_version: 'ubuntu_22_x86' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse3-dev' @@ -36,7 +36,7 @@ jobs: adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) Ubuntu-22-ARM64: AgentName: 'blobfuse-ubn22-arm64' - distro: ubuntu + distro: 'ubuntu' distro_version: 'ubuntu_22_arm' poolName: 'blobfuse-ubn-arm64-pool' fuselib: 'libfuse3-dev' From 7ea5b25e99a88d09e0ced5d407ef584cc060ce8e Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 16:53:34 +0000 Subject: [PATCH 46/87] Modify package install --- azure-pipeline-templates/package-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index d16a05a57..898dd9a63 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -22,7 +22,7 @@ steps: - ${{ if eq(parameters.distro, 'ubuntu') }}: - script: | echo "these are equal" - displayName: "these are equal" + displayName: 'these are equal' - ${{ if ne(parameters.distro, 'ubuntu') }}: - script: | sleep 30 From 2ff24362d9a1a6ac1b5939fcd07c72ba8c313cc7 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 16:54:44 +0000 Subject: [PATCH 47/87] Modify package install --- azure-pipeline-templates/package-install.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 898dd9a63..2fa3b18cb 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -22,7 +22,7 @@ steps: - ${{ if eq(parameters.distro, 'ubuntu') }}: - script: | echo "these are equal" - displayName: 'these are equal' + displayName: 'these are equal' - ${{ if ne(parameters.distro, 'ubuntu') }}: - script: | sleep 30 From 7c25d2a8fc45ba6c45314a965ff1eadccb1a4825 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Sun, 16 Feb 2025 17:47:15 +0000 Subject: [PATCH 48/87] Modify package install --- azure-pipeline-templates/build.yml | 4 +- azure-pipeline-templates/container.yml | 15 +- .../generate-container-name.yml | 6 - azure-pipeline-templates/package-install.yml | 182 ++++++++---------- 4 files changed, 101 insertions(+), 106 deletions(-) delete mode 100644 azure-pipeline-templates/generate-container-name.yml diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 922a3ef10..6f1d8cd81 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -139,7 +139,9 @@ steps: # Create Container in storage Account. # Include the template to generate the container name # This will set the value generated to the global containerName parameter for the job. - - template: 'generate-container-name.yml' + - template: 'container.yml' + parameters: + generate_container: true # Create a Container in the block storage account - template: 'container.yml' diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index b7c0931d1..b51fa1586 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -1,5 +1,8 @@ # Responsible for creation and deletion of container. parameters: + - name: generate_container + type: boolean + default: false - name: create_container type: boolean default: false @@ -8,15 +11,25 @@ parameters: default: false - name: account_type # accepted values: block, adls type: string + default: invalid - name: account_name type: string + default: invalid - name: account_key type: string + default: invalid - name: container_name type: string - + default: invalid steps: + # Generate the Container Name + - ${{ if eq(parameters.generate_container, true) }}: + - bash: | + CONTAINER_NAME=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) + echo "Generated container name: $CONTAINER_NAME" + echo "##vso[task.setvariable variable=containerName]$CONTAINER_NAME" + displayName: 'Generate random container name' # Creating the Container - ${{ if eq(parameters.create_container, true) }}: - script: | diff --git a/azure-pipeline-templates/generate-container-name.yml b/azure-pipeline-templates/generate-container-name.yml deleted file mode 100644 index 79a6bc15b..000000000 --- a/azure-pipeline-templates/generate-container-name.yml +++ /dev/null @@ -1,6 +0,0 @@ -steps: - - bash: | - CONTAINER_NAME=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) - echo "Generated container name: $CONTAINER_NAME" - echo "##vso[task.setvariable variable=containerName]$CONTAINER_NAME" - displayName: 'Generate random container name' diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 2fa3b18cb..d27d4c99d 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -1,113 +1,99 @@ -parameters: - - name: distro - type: string - default: ubuntu - # values: - # - ubuntu - # - rhel - # - centos - # - oracle - # - rocky - # - suse - # - mariner - - name: distro_version - type: string - default: ubuntu +# Supported Distros +# - ubuntu +# - rhel +# - centos +# - oracle +# - rocky +# - suse +# - mariner steps: - script: | - echo ${{ parameters.distro }} - displayName: 'Check on something' - - - ${{ if eq(parameters.distro, 'ubuntu') }}: - - script: | - echo "these are equal" - displayName: 'these are equal' - - ${{ if ne(parameters.distro, 'ubuntu') }}: - - script: | - sleep 30 - sudo systemctl stop apt-daily.timer - echo "Add Lock Timeout for apt package manager" - sudo sh -c 'echo "DPkg::Lock::Timeout \"120\";" > /etc/apt/apt.conf.d/99timeout' - sudo ps -aux | grep -iE "apt" - sudo killall apt apt-get - yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock - echo "Released any lock if some other process has acquired" - sudo dpkg --configure -a - echo "****************Starting Updates and Installation of Packages**************" - sudo apt-get update --fix-missing - sudo apt-get install make cmake gcc g++ git parallel -y - if [ "$(tags)" == "fuse2" ] - then - sudo apt-get install libfuse2 libfuse-dev -y - else - sudo apt-get install fuse3 libfuse3-dev -y - fi - fusermount3 -V - fusermount -V - echo "***********************Install AzCli*************************" - curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash - az --version - displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + sleep 30 + sudo systemctl stop apt-daily.timer + echo "Add Lock Timeout for apt package manager" + sudo sh -c 'echo "DPkg::Lock::Timeout \"120\";" > /etc/apt/apt.conf.d/99timeout' + sudo ps -aux | grep -iE "apt" + sudo killall apt apt-get + yes | sudo fuser -vik -TERM /var/lib/dpkg/lock /var/lib/dpkg/lock-frontend /var/lib/apt/lists/lock + echo "Released any lock if some other process has acquired" + sudo dpkg --configure -a + echo "****************Starting Updates and Installation of Packages**************" + sudo apt-get update --fix-missing + sudo apt-get install make cmake gcc g++ git parallel -y + if [ "$(tags)" == "fuse2" ] + then + sudo apt-get install libfuse2 libfuse-dev -y + else + sudo apt-get install fuse3 libfuse3-dev -y + fi + fusermount3 -V + fusermount -V + echo "***********************Install AzCli*************************" + curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash + az --version + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + condition: eq(variables['distro'], 'ubuntu') # RHEL PACKAGE INSTALLATION - - ${{ if eq(parameters.distro, 'rhel') }}: - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo yum update -y - sudo yum groupinstall "Development Tools" -y - if [ $(AgentName) == "blobfuse-rhel9" ]; then - sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing - else - sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing - fi - displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo yum update -y + sudo yum groupinstall "Development Tools" -y + if [ $(AgentName) == "blobfuse-rhel9" ]; then + sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing + else + sudo yum install git fuse fuse3-libs fuse3-devel fuse3 python36 -y --nobest --allowerasing + fi + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + condition: eq(variables['distro'], 'rhel') # CENTOS PACKAGE INSTALLATION - - ${{ if eq(parameters.distro, 'centos') }}: - - script: | - if [ "$(AgentName)" == "blobfuse-centos8" ] - then - sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* - sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* - fi - sudo yum update -y --skip-broken - if [ $(AgentName) == "blobfuse-centos8" ]; then - sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing - else - sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y - fi - displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + - script: | + if [ "$(AgentName)" == "blobfuse-centos8" ] + then + sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* + sudo sed -i 's|baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* + fi + sudo yum update -y --skip-broken + if [ $(AgentName) == "blobfuse-centos8" ]; then + sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing + else + sudo yum install gcc gcc-c++ make git fuse3 fuse3-devel python36 -y + fi + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + condition: eq(variables['distro'], 'centos') # ORACLE PACKAGE INSTALLATION - - ${{ if eq(parameters.distro, 'oracle') }}: - - script: | - sudo yum update -y - sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing - displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + - script: | + sudo yum update -y + sudo yum install gcc gcc-c++ make git fuse fuse3 fuse3-devel python36 -y --nobest --allowerasing + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + condition: eq(variables['distro'], 'oracle') # ROCKY PACKAGE INSTALLATION - - ${{ if eq(parameters.distro, 'rocky') }}: - - script: | - sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo - sudo yum update -y - sudo yum groupinstall "Development Tools" -y - sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing - displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + - script: | + sudo sed -i '/^failovermethod=/d' /etc/yum.repos.d/*.repo + sudo yum update -y + sudo yum groupinstall "Development Tools" -y + sudo yum install wget git fuse fuse3-libs fuse3-devel fuse3 python3 -y --nobest --allowerasing + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + condition: eq(variables['distro'], 'rocky') # SUSE PACKAGE INSTALLATION - - ${{ if eq(parameters.distro, 'suse') }}: - - script: | - sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse - wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm - fusermount3 --v - wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + - script: | + sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse + wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm + sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm + fusermount3 --v + wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm + sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + condition: eq(variables['distro'], 'suse') # MARINER PACKAGE INSTALLATION - - ${{ if eq(parameters.distro, 'mariner') }}: - - script: | - sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y - displayName: 'Libfuse Setup ${{ parameters.distro_version }}' \ No newline at end of file + - script: | + sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y + displayName: 'Libfuse Setup ${{ parameters.distro_version }}' + condition: eq(variables['distro'], 'mariner') \ No newline at end of file From 428727723e25f443a332236d6cb9b60e02fc3191 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 06:25:21 +0000 Subject: [PATCH 49/87] Changes for nightly pipeline --- azure-pipeline-templates/build.yml | 2 +- azure-pipeline-templates/container.yml | 3 +- azure-pipeline-templates/distro-tests.yml | 95 -------------- .../e2e-tests-block-cache-data-integrity.yml | 5 +- .../e2e-tests-block-cache.yml | 5 +- azure-pipeline-templates/e2e-tests-spcl.yml | 22 +--- azure-pipeline-templates/e2e-tests.yml | 15 +-- azure-pipeline-templates/mount-test.yml | 24 +--- azure-pipeline-templates/setup.yml | 94 ------------- azure-pipeline-templates/stress-test.yml | 25 ++-- azure-pipeline-templates/verbose-tests.yml | 34 ----- azure-pipeline-templates/verify-auth.yml | 3 - blobfuse2-code-coverage.yaml | 1 - blobfuse2-nightly.yaml | 124 +++++++++--------- 14 files changed, 94 insertions(+), 358 deletions(-) delete mode 100755 azure-pipeline-templates/distro-tests.yml delete mode 100644 azure-pipeline-templates/setup.yml diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 6f1d8cd81..3a05e2439 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -2,7 +2,7 @@ # 1. Setup the VM all necessary packages. # 2. Build blobfuse, bfusemon, install AzCLI # 3. Create Temporary Containers in the storage account with choosing random string as name and also sets the global containerName paramter to this value. -# Caution: The Deletion of the container is left to the caller of this file, caller must call the cleanup.yml with the parameter delete_containers: true +# Caution: The Deletion of the container is left to the caller of this file, caller must call the cleanup.yml with the parameter delete_containers: true at the end of the job # 4. Run Unit Tests if specified by parameter skip_ut: false parameters: diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index b51fa1586..9ba6a3b31 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -49,4 +49,5 @@ steps: az storage container delete --name ${{ parameters.container_name }} \ --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ - displayName: 'Deleting ${{ parameters.account_type }} Container' \ No newline at end of file + displayName: 'Deleting ${{ parameters.account_type }} Container' + condition: always() \ No newline at end of file diff --git a/azure-pipeline-templates/distro-tests.yml b/azure-pipeline-templates/distro-tests.yml deleted file mode 100755 index c108c1307..000000000 --- a/azure-pipeline-templates/distro-tests.yml +++ /dev/null @@ -1,95 +0,0 @@ -parameters: - - name: config_path - type: string - - name: blob_account_name - type: string - - name: blob_account_key - type: string - - name: adls_account_name - type: string - - name: adls_account_key - type: string - - name: distro_name - type: string - - name: quick_test - type: boolean - default: true - - name: verbose_log - type: boolean - default: false - - name: clone - type: boolean - default: false - -steps: - # Get the host details on which these test are running - - script: | - echo $(Description) - hostnamectl - displayName: 'Print Agent Info' - - - checkout: none - - - script: | - sudo rm -rf $(ROOT_DIR) - sudo mkdir -p $(ROOT_DIR) - sudo chown -R `whoami` $(ROOT_DIR) - chmod 777 $(ROOT_DIR) - mkdir -p $(ROOT_DIR)/azure-storage-fuse - displayName: 'Create Directory Structure' - - # Build the code - - template: 'build.yml' - parameters: - distro_name: $(DistroVer) - distro_id: $(distroId) - - # Block Blob Test - - template: 'e2e-tests-spcl.yml' - parameters: - conf_template: azure_key.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: block_blob_key - adls: false - account_name: ${{ parameters.blob_account_name }} - account_key: ${{ parameters.blob_account_key }} - account_type: block - account_endpoint: https://${{ parameters.blob_account_name }}.blob.core.windows.net - distro_name: ${{ parameters.distro_name }} - quick_test: ${{ parameters.quick_test }} - verbose_log: ${{ parameters.verbose_log }} - clone: ${{ parameters.clone }} - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - - # ADLS Test - - template: 'e2e-tests-spcl.yml' - parameters: - conf_template: azure_key.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: adls_key - adls: true - account_name: ${{ parameters.adls_account_name }} - account_key: ${{ parameters.adls_account_key }} - account_type: adls - account_endpoint: https://${{ parameters.adls_account_name }}.dfs.core.windows.net - distro_name: ${{ parameters.distro_name }} - quick_test: ${{ parameters.quick_test }} - verbose_log: ${{ parameters.verbose_log }} - clone: ${{ parameters.clone }} - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - - # Cleanup go tools dir - - task: GO@0 - inputs: - command: 'clean' - arguments: '-cache -modcache -testcache -i' - workingDirectory: $(WORK_DIR) - displayName: 'Clean up go tool dirs' - timeoutInMinutes: 20 - continueOnError: true - condition: always() - diff --git a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml index 5558ce2d8..ef5292af5 100644 --- a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml +++ b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml @@ -211,6 +211,5 @@ steps: - template: 'cleanup.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} \ No newline at end of file + unmount: true + delete_containers: false \ No newline at end of file diff --git a/azure-pipeline-templates/e2e-tests-block-cache.yml b/azure-pipeline-templates/e2e-tests-block-cache.yml index 3bcdae4e1..ccf39f679 100644 --- a/azure-pipeline-templates/e2e-tests-block-cache.yml +++ b/azure-pipeline-templates/e2e-tests-block-cache.yml @@ -267,6 +267,5 @@ steps: - template: 'cleanup.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} \ No newline at end of file + unmount: true + delete_containers: false \ No newline at end of file diff --git a/azure-pipeline-templates/e2e-tests-spcl.yml b/azure-pipeline-templates/e2e-tests-spcl.yml index 470ea8bc9..9fff1b481 100644 --- a/azure-pipeline-templates/e2e-tests-spcl.yml +++ b/azure-pipeline-templates/e2e-tests-spcl.yml @@ -2,13 +2,7 @@ parameters: - name: conf_template type: string - name: config_file - type: string - - name: container - type: string - - name: temp_dir - type: string - - name: mount_dir - type: string + type: string - name: idstring type: string - name: adls @@ -19,7 +13,6 @@ parameters: type: string - name: account_type type: string - - name: account_endpoint - name: distro_name type: string - name: quick_test @@ -38,13 +31,12 @@ parameters: steps: - script: | cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=${{ parameters.conf_template }} --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=${{ parameters.conf_template }} --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} displayName: 'Create Config File : ${{ parameters.idstring }}' env: NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -54,9 +46,6 @@ steps: - template: './e2e-tests.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} adls: ${{ parameters.adls }} idstring: '${{ parameters.idstring }}' distro_name: ${{ parameters.distro_name }} @@ -67,13 +56,12 @@ steps: enable_symlink_adls: ${{ parameters.enable_symlink_adls }} mountStep: script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(System.DefaultWorkingDirectory) + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(System.DefaultWorkingDirectory) displayName: 'E2E TEST : ${{ parameters.idstring }}' timeoutInMinutes: 3 continueOnError: false - template: 'cleanup.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} \ No newline at end of file + unmount: true + delete_containers: false \ No newline at end of file diff --git a/azure-pipeline-templates/e2e-tests.yml b/azure-pipeline-templates/e2e-tests.yml index bbf6e6f37..23c45329a 100755 --- a/azure-pipeline-templates/e2e-tests.yml +++ b/azure-pipeline-templates/e2e-tests.yml @@ -1,10 +1,4 @@ parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string - name: idstring type: string - name: distro_name @@ -33,9 +27,6 @@ steps: # Mount the container - template: 'mount.yml' parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} mountStep: ${{ parameters.mountStep }} @@ -66,13 +57,11 @@ steps: - task: Go@0 inputs: command: 'test' - arguments: '-v -timeout=2h ./... -args -mnt-path=${{ parameters.mount_dir }} -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=${{parameters.temp_dir}} -quick-test=${{parameters.quick_test}} -enable-symlink-adls=${{parameters.enable_symlink_adls}} -distro-name="${{parameters.distro_name}}"' - workingDirectory: ${{ parameters.working_dir }}/test/e2e_tests + arguments: '-v -timeout=2h ./... -args -mnt-path=$(MOUNT_DIR) -adls=${{parameters.adls}} -clone=${{parameters.clone}} -tmp-path=$(TEMP_DIR) -quick-test=${{parameters.quick_test}} -enable-symlink-adls=${{parameters.enable_symlink_adls}} -distro-name="${{parameters.distro_name}}"' + workingDirectory: $(WORK_DIR)/test/e2e_tests displayName: 'E2E Test: ${{ parameters.idstring }}' timeoutInMinutes: 120 continueOnError: false - env: - mount_dir: ${{ parameters.mount_dir }} # Publish build artifacts # Publish build artifacts to Azure Pipelines or a Windows file share diff --git a/azure-pipeline-templates/mount-test.yml b/azure-pipeline-templates/mount-test.yml index 779dc7534..0adb731e6 100644 --- a/azure-pipeline-templates/mount-test.yml +++ b/azure-pipeline-templates/mount-test.yml @@ -1,38 +1,29 @@ parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - name: config type: string - - name: temp_dir - type: string - name: idstring type: string default: '' - - name: tags - type: string steps: # Cleanup - template: 'cleanup.yml' parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + unmount: true + delete_containers: false # Run the mount test script - task: Go@0 inputs: command: 'test' - arguments: '-timeout=120m -p 1 -v test/mount_test/mount_test.go -args -working-dir=${{ parameters.working_dir }} -mnt-path=${{ parameters.mount_dir }} -config-file=${{parameters.config}} -tags=${{ parameters.tags }}' - workingDirectory: ${{ parameters.working_dir }} + arguments: '-timeout=120m -p 1 -v test/mount_test/mount_test.go -args -working-dir=$(WORK_DIR) -mnt-path=$(MOUNT_DIR) -config-file=${{parameters.config}} -tags=$(tags)' + workingDirectory: $(WORK_DIR) displayName: 'MountTest: ${{ parameters.idstring }}' timeoutInMinutes: 120 continueOnError: true env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) - script: | cat blobfuse2-logs.txt @@ -45,6 +36,5 @@ steps: # Cleanup - template: 'cleanup.yml' parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} \ No newline at end of file + unmount: true + delete_containers: false diff --git a/azure-pipeline-templates/setup.yml b/azure-pipeline-templates/setup.yml deleted file mode 100644 index 5cdd9d984..000000000 --- a/azure-pipeline-templates/setup.yml +++ /dev/null @@ -1,94 +0,0 @@ -parameters: - - name: installStep - type: step - - name: tags - type: string - default: "null" - -steps: - # Create directory structure and prepare to mount - - ${{ parameters.installStep }} - - checkout: none - - # Get the host details on which these test are running - - script: | - echo $(Description) - hostnamectl - displayName: 'Print Agent Info' - - - script: | - sudo rm -rf $(ROOT_DIR) - sudo mkdir -p $(ROOT_DIR) - sudo chown -R `whoami` $(ROOT_DIR) - chmod 777 $(ROOT_DIR) - mkdir -p $(ROOT_DIR)/go/src - displayName: 'Create Directory Structure' - - # Clone the repo - - script: | - git clone https://github.com/Azure/azure-storage-fuse - displayName: 'Checkout Code' - workingDirectory: $(ROOT_DIR)/go/src - - # Checkout the branch - - script: | - git checkout `echo $(Build.SourceBranch) | cut -d "/" -f 1,2 --complement` - displayName: 'Checkout Branch' - workingDirectory: $(WORK_DIR) - - # Custom script to install Go-lang - - task: ShellScript@2 - inputs: - scriptPath: "$(WORK_DIR)/go_installer.sh" - args: "$(ROOT_DIR)/" - displayName: "GoTool Custom Setup" - - # Downloading Go dependency packages - - task: Go@0 - inputs: - command: 'get' - arguments: '-d' - workingDirectory: $(WORK_DIR) - displayName: "Go Get" - - # Building our package - - task: Go@0 - inputs: - command: 'build' - arguments: "-tags ${{ parameters.tags }} -o blobfuse2" - workingDirectory: $(WORK_DIR) - displayName: "Go Build" - - # Building our package - - task: Go@0 - inputs: - command: 'build' - arguments: "-tags ${{ parameters.tags }} -o bfusemon ./tools/health-monitor/" - workingDirectory: $(WORK_DIR) - displayName: "Go Build bfusemon" - - #Verification of built binary - - script: | - sudo chmod +x $(WORK_DIR)/blobfuse2 - $(WORK_DIR)/blobfuse2 --version - displayName: 'Test binary' - env: - working_directory: $(WORK_DIR) - - # Creating necessary directories - - script: | - sudo fusermount -u $(MOUNT_DIR) - sudo fusermount3 -u $(MOUNT_DIR) - rm -rf $(MOUNT_DIR) - mkdir -p $(MOUNT_DIR) - echo "Creating mount dir " $(MOUNT_DIR) - displayName: 'Create Mount directory' - env: - mount_dir: $(MOUNT_DIR) - - - script: | - mkdir -p $(TEMP_DIR) - echo "Creating temp dir " $(TEMP_DIR) - displayName: "Create Cache directory" - env: - temp_dir: $(TEMP_DIR) diff --git a/azure-pipeline-templates/stress-test.yml b/azure-pipeline-templates/stress-test.yml index d598ffcfe..08b3ff5f4 100755 --- a/azure-pipeline-templates/stress-test.yml +++ b/azure-pipeline-templates/stress-test.yml @@ -1,10 +1,4 @@ parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string - name: stress_dir type: string - name: idstring @@ -25,9 +19,9 @@ steps: # Mount the container - template: 'mount.yml' parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + working_dir: $(WORK_DIR) + mount_dir: $(MOUNT_DIR) + temp_dir: $(TEMP_DIR) prefix: ${{ parameters.idstring }} mountStep: ${{ parameters.mountStep }} @@ -35,13 +29,13 @@ steps: - task: Go@0 inputs: command: 'test' - arguments: '-timeout 120m -v test/stress_test/stress_test.go -args -mnt-path=${{ parameters.mount_dir }} -quick=${{parameters.quick}}' - workingDirectory: ${{ parameters.working_dir }} + arguments: '-timeout 120m -v test/stress_test/stress_test.go -args -mnt-path=$(MOUNT_DIR) -quick=${{parameters.quick}}' + workingDirectory: $(WORK_DIR) displayName: 'StressTest: ${{ parameters.idstring }}' timeoutInMinutes: 120 continueOnError: true env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) # Delete all files test - script: rm -rf ${mount_dir}/* @@ -49,7 +43,7 @@ steps: displayName: 'StressTest: ${{ parameters.idstring }} Cleanup' continueOnError: true env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) - script: | cat blobfuse2-logs.txt @@ -62,8 +56,7 @@ steps: # Cleanup - template: 'cleanup.yml' parameters: - working_dir: ${{ parameters.working_dir }} - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} + unmount: true + delete_containers: false diff --git a/azure-pipeline-templates/verbose-tests.yml b/azure-pipeline-templates/verbose-tests.yml index 13675805a..764ba14b5 100644 --- a/azure-pipeline-templates/verbose-tests.yml +++ b/azure-pipeline-templates/verbose-tests.yml @@ -132,9 +132,6 @@ steps: - ${{ if eq(parameters.test_key_credential, true) }}: - template: e2e-tests.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with Key Credentials distro_name: ${{ parameters.distro_name }} @@ -155,9 +152,6 @@ steps: parameters: conf_template: azure_block_perf.yaml config_file: ${{ parameters.config }} - container: $(containerName) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -172,9 +166,6 @@ steps: - ${{ if eq(parameters.test_sas_credential, true) }}: - template: e2e-tests.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with SAS Credentials distro_name: ${{ parameters.distro_name }} @@ -191,9 +182,6 @@ steps: - ${{ if eq(parameters.test_azcli_credential, true) }}: - template: e2e-tests.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with Azure CLI Credentials distro_name: ${{ parameters.distro_name }} @@ -218,9 +206,6 @@ steps: - template: e2e-tests.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) adls: ${{ parameters.adls }} idstring: ${{ parameters.service }} with Azurite distro_name: ${{ parameters.distro_name }} @@ -241,9 +226,6 @@ steps: parameters: conf_template: azure_key_lru_purge.yaml config_file: ${{ parameters.config }} - container: $(containerName) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -257,9 +239,6 @@ steps: parameters: conf_template: azure_key_emptyfile.yaml config_file: ${{ parameters.config }} - container: $(containerName) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -275,9 +254,6 @@ steps: parameters: conf_template: azure_key_directio.yaml config_file: ${{ parameters.config }} - container: $(containerName) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -293,9 +269,6 @@ steps: parameters: conf_template: azure_key_symlink.yaml config_file: ${{ parameters.config }} - container: $(containerName) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) adls: ${{ parameters.adls }} account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} @@ -325,18 +298,11 @@ steps: - template: mount-test.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) config: ${{ parameters.config }} idstring: ${{ parameters.service }} Mount Test - tags: $(tags) - template: stress-test.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) stress_dir: ${{ parameters.stress_dir }} idstring: ${{ parameters.service }} Stress Test quick: '${{ parameters.quick_stress }}' diff --git a/azure-pipeline-templates/verify-auth.yml b/azure-pipeline-templates/verify-auth.yml index b6e3b63ae..962500ecb 100755 --- a/azure-pipeline-templates/verify-auth.yml +++ b/azure-pipeline-templates/verify-auth.yml @@ -10,9 +10,6 @@ steps: # Mount the container - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) prefix: ${{ parameters.idstring }} mountStep: ${{ parameters.mountStep }} diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 67488af6b..96f99bc41 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -575,4 +575,3 @@ stages: parameters: unmount: false delete_containers: true - condition: always() diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 2988c8427..195350507 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -21,7 +21,7 @@ parameters: - name: exhaustive_test # Quick test or an exhaustive test - displayName: 'Exhaustive test' + displayName: 'Run E2E tests on all distros' type: boolean default: true @@ -37,11 +37,6 @@ parameters: type: boolean default: true - - name: healthmon_test - displayName: 'Healthmon test' - type: boolean - default: true - - name: quick_stress displayName: 'Quick Stress' type: boolean @@ -321,9 +316,6 @@ stages: # End to End tests - template: 'azure-pipeline-templates/e2e-tests.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) idstring: 'BlockBlob with Proxy and Key Credentials' distro_name: $(AgentName) adls: false @@ -355,9 +347,6 @@ stages: - template: 'azure-pipeline-templates/e2e-tests.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) idstring: 'ADLS with Proxy and Key Credentials' distro_name: $(AgentName) adls: true @@ -529,59 +518,59 @@ stages: strategy: matrix: RHEL-8.6: - DistroVer: 'RHEL-8.6' + distro_version: 'RHEL-8.6' distro: rhel poolName: 'blobfuse-rhel-pool' Description: 'Red Hat Enterprise Linux 8.6' AgentName: 'blobfuse-rhel8_6' tags: 'fuse3' RHEL-9.0: - DistroVer: 'RHEL-9.0' + distro_version: 'RHEL-9.0' distro: rhel poolName: 'blobfuse-rhel-pool' Description: 'Red Hat Enterprise Linux 9.0' AgentName: 'blobfuse-rhel9' tags: 'fuse3' CentOS-7.9: - DistroVer: "CentOS-7.9" + distro_version: "CentOS-7.9" distro: centos poolName: 'blobfuse-centos-pool' Description: "CentOS 7.9" AgentName: "blobfuse-centos7" CentOS-8.5: - DistroVer: "CentOS-8.5" + distro_version: "CentOS-8.5" distro: centos poolName: 'blobfuse-centos-pool' Description: "CentOS 8.5" AgentName: "blobfuse-centos8" Oracle-8.1: - DistroVer: "Oracle-8.1" + distro_version: "Oracle-8.1" distro: oracle poolName: 'blobfuse-oracle-pool' Description: "Oracle Linux 8.1" AgentName: "blobfuse-oracle81" Rocky-8.0: - DistroVer: "Rocky-8.0" + distro_version: "Rocky-8.0" distro: rocky poolName: 'blobfuse2-rocky-pool' Description: "Rocky Linux 8.0" AgentName: "blobfuse-rocky8" tags: 'fuse3' Rocky-9.0: - DistroVer: "Rocky-9.0" + distro_version: "Rocky-9.0" distro: rocky poolName: 'blobfuse2-rocky-pool' Description: "Rocky Linux 9.0" AgentName: "blobfuse-rocky9" tags: 'fuse3' SUSE-15: - DistroVer: "SUSE-15" + distro_version: "SUSE-15" distro: suse poolName: 'blobfuse-suse-pool' Description: "SUSE Enterprise Linux 15" AgentName: "blobfuse-suse15" Mariner2: - DistroVer: "Mariner2" + distro_version: "Mariner2" distro: mariner poolName: 'blobfuse-mariner-pool' Description: "CBL-Mariner2 Linux" @@ -613,22 +602,45 @@ stages: value: true steps: - - template: 'azure-pipeline-templates/distro-tests.yml' + # Build the code + - template: 'build.yml' + + # Block Blob Test + - template: 'e2e-tests-spcl.yml' parameters: - working_dir: $(WORK_DIR) - root_dir: $(ROOT_DIR) - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - config_path: $(BLOBFUSE2_CFG) - container: $(ContainerName) - blob_account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - blob_account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - adls_account_name: $(AZTEST_ADLS_ACC_NAME) - adls_account_key: $(AZTEST_ADLS_KEY) - distro_name: $(AgentName) - gopath: $(GOPATH) - tags: $(tags) + conf_template: azure_key.yaml + config_file: $(BLOBFUSE2_CFG) + idstring: block_blob_key + adls: false + account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) + account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) + account_type: block + distro_name: $(distro_version) + quick_test: true + verbose_log: ${{ parameters.verbose_log }} + clone: false + + # ADLS Test + - template: 'e2e-tests-spcl.yml' + parameters: + conf_template: azure_key.yaml + config_file: $(BLOBFUSE2_CFG) + idstring: adls_key + adls: true + account_name: $(AZTEST_ADLS_ACC_NAME) + account_key: $(AZTEST_ADLS_KEY) + account_type: adls + distro_name: $(distro_version) + quick_test: true verbose_log: ${{ parameters.verbose_log }} + clone: false + + # Cleanup + - template: 'azure-pipeline-templates/cleanup.yml' + parameters: + unmount: true + delete_containers: true + - stage: BlockCacheDataValidation jobs: @@ -679,14 +691,6 @@ stages: # Pull and build the code - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) skip_ut: true - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' @@ -707,7 +711,12 @@ stages: # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable temp_dir: $(TEMP_DIR) mount_dir: $(MOUNT_DIR) - + + - template: azure-pipeline-templates/cleanup.yml + parameters: + unmount: true + delete_containers: true + - stage: BlockCacheDataDirectIOValidation jobs: # Ubuntu Tests @@ -757,14 +766,6 @@ stages: # Pull and build the code - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) skip_ut: true - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' @@ -787,6 +788,12 @@ stages: mount_dir: $(MOUNT_DIR) mnt_flags: "-o direct_io" + - template: azure-pipeline-templates/cleanup.yml + parameters: + unmount: true + delete_containers: true + + - stage: BlockCacheDataIntegrityValidation jobs: @@ -837,14 +844,6 @@ stages: # Pull and build the code - template: 'azure-pipeline-templates/build.yml' parameters: - working_directory: $(WORK_DIR) - root_dir: $(ROOT_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) - gopath: $(GOPATH) - container: $(containerName) - tags: $(tags) - fuselib: $(fuselib) skip_ut: true - script: | @@ -893,6 +892,11 @@ stages: mount_dir: $(MOUNT_DIR) block_size_mb: "8" + - template: azure-pipeline-templates/cleanup.yml + parameters: + unmount: true + delete_containers: true + # Run e2e tests for file cache, stream on hns, fns From 5b0e9d5072d9fa7a73d9eb36298d928aff3c0844 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 06:38:39 +0000 Subject: [PATCH 50/87] Changes for nightly pipeline --- azure-pipeline-templates/verbose-tests.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/azure-pipeline-templates/verbose-tests.yml b/azure-pipeline-templates/verbose-tests.yml index 764ba14b5..2535b815f 100644 --- a/azure-pipeline-templates/verbose-tests.yml +++ b/azure-pipeline-templates/verbose-tests.yml @@ -156,7 +156,6 @@ steps: account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} - account_endpoint: ${{ parameters.account_endpoint }} idstring: "${{ parameters.service }} with Block-cache" distro_name: ${{ parameters.distro_name }} quick_test: false @@ -230,7 +229,6 @@ steps: account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} - account_endpoint: ${{ parameters.account_endpoint }} idstring: "${{ parameters.service }} LRU policy no timeout" distro_name: ${{ parameters.distro_name }} verbose_log: ${{ parameters.verbose_log }} @@ -243,7 +241,6 @@ steps: account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} - account_endpoint: ${{ parameters.account_endpoint }} idstring: "${{ parameters.service }} LRU policy create empty" distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} @@ -258,7 +255,6 @@ steps: account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} - account_endpoint: ${{ parameters.account_endpoint }} idstring: "${{ parameters.service }} Direct IO tests" distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} @@ -273,7 +269,6 @@ steps: account_name: ${{ parameters.account_name }} account_key: ${{ parameters.account_key }} account_type: ${{ parameters.account_type }} - account_endpoint: ${{ parameters.account_endpoint }} idstring: "${{ parameters.service }} Symlink config tests" distro_name: ${{ parameters.distro_name }} quick_test: ${{ parameters.quick_test }} From 66c612d89e400af91111f058aafb9a0dac71436d Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 06:42:06 +0000 Subject: [PATCH 51/87] Changes for nightly pipeline --- .../e2e-tests-block-cache-data-integrity.yml | 9 --------- .../e2e-tests-block-cache.yml | 15 --------------- azure-pipeline-templates/stress-test.yml | 3 --- 3 files changed, 27 deletions(-) diff --git a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml index ef5292af5..32bfa74bf 100644 --- a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml +++ b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml @@ -53,9 +53,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} mountStep: script: | @@ -99,9 +96,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} ro_mount: true mountStep: @@ -155,9 +149,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} ro_mount: true mountStep: diff --git a/azure-pipeline-templates/e2e-tests-block-cache.yml b/azure-pipeline-templates/e2e-tests-block-cache.yml index ccf39f679..592c27d8d 100644 --- a/azure-pipeline-templates/e2e-tests-block-cache.yml +++ b/azure-pipeline-templates/e2e-tests-block-cache.yml @@ -54,9 +54,6 @@ steps: # run below step only if direct_io is false - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} mountStep: script: | @@ -98,9 +95,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} ro_mount: true mountStep: @@ -129,9 +123,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} ro_mount: true mountStep: @@ -168,9 +159,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} ro_mount: true mountStep: @@ -220,9 +208,6 @@ steps: - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: ${{ parameters.mount_dir }} - temp_dir: ${{ parameters.temp_dir }} prefix: ${{ parameters.idstring }} ro_mount: true mountStep: diff --git a/azure-pipeline-templates/stress-test.yml b/azure-pipeline-templates/stress-test.yml index 08b3ff5f4..f30c8eb7d 100755 --- a/azure-pipeline-templates/stress-test.yml +++ b/azure-pipeline-templates/stress-test.yml @@ -19,9 +19,6 @@ steps: # Mount the container - template: 'mount.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) prefix: ${{ parameters.idstring }} mountStep: ${{ parameters.mountStep }} From a8b28e8e04cf6ecc03aab0c3742a2775e980cc20 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 06:50:52 +0000 Subject: [PATCH 52/87] Changes for nightly pipeline --- blobfuse2-nightly.yaml | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 195350507..150d81893 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -60,16 +60,22 @@ stages: matrix: Ubuntu-20-BlockBlob: AgentName: 'blobfuse-ubuntu20' + distro: 'ubuntu' + distro_version: 'ubuntu_20' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22-BlockBlob: AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' poolName: 'blobfuse-ubuntu-pool' fuselib: 'libfuse3-dev' tags: 'fuse3' Ubuntu-22-ARM64-BlockBlob: AgentName: 'blobfuse-ubn22-arm64' + distro: 'ubuntu' + distro_version: 'ubuntu_22_arm64' poolName: 'blobfuse-ubn-arm64-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM) fuselib: 'libfuse3-dev' @@ -160,18 +166,24 @@ stages: matrix: Ubuntu-20-ADLS: AgentName: 'blobfuse-ubuntu20' + distro: 'ubuntu' + distro_version: 'ubuntu_20' poolName: 'blobfuse-ubuntu-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22-ADLS: AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' poolName: 'blobfuse-ubuntu-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' Ubuntu-22-ARM64-ADLS: AgentName: 'blobfuse-ubn22-arm64' + distro: 'ubuntu' + distro_version: 'ubuntu_22_arm64' poolName: 'blobfuse-ubn-arm64-pool' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM) fuselib: 'libfuse3-dev' @@ -262,6 +274,8 @@ stages: matrix: Ubuntu-20-Proxy: AgentName: 'blobfuse-ubuntu20' + distro: 'ubuntu' + distro_version: 'ubuntu_20' pool: name: "blobfuse-ubuntu-pool" @@ -379,9 +393,6 @@ stages: - template: 'azure-pipeline-templates/verify-auth.yml' parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) idstring: 'Block SAS' distro_name: $(AgentName) mountStep: @@ -407,6 +418,8 @@ stages: matrix: Ubuntu-20-MSI: DistroVer: "Ubn20_MSI" + distro: 'ubuntu' + distro_version: 'ubuntu_20' AgentName: "blobfuse-ubuntu20" Description: "Ubuntu 20 MSI Test" @@ -651,6 +664,8 @@ stages: matrix: Ubuntu-22: AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' containerName: 'test-cnt-ubn-22' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' @@ -726,12 +741,16 @@ stages: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' + distro: 'ubuntu' + distro_version: 'ubuntu_20_x86' containerName: 'test-cnt-ubn-20' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22: AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' containerName: 'test-cnt-ubn-22' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' @@ -804,12 +823,16 @@ stages: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' + distro: 'ubuntu' + distro_version: 'ubuntu_20_x86' containerName: 'test-cnt-ubn-20' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22: AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' containerName: 'test-cnt-ubn-22' adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' From 54b0a949707319e420ed8c87d57efa3be5df5206 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 06:52:38 +0000 Subject: [PATCH 53/87] Changes for nightly pipeline --- blobfuse2-nightly.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 150d81893..5ef04b0e6 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -616,7 +616,7 @@ stages: steps: # Build the code - - template: 'build.yml' + - template: 'azure-pipeline-templates/build.yml' # Block Blob Test - template: 'e2e-tests-spcl.yml' From 5a6a472b97dde25e3498b089bd08fc9f6408835f Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 06:53:39 +0000 Subject: [PATCH 54/87] Changes for nightly pipeline --- blobfuse2-nightly.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 5ef04b0e6..0f92dcfac 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -619,7 +619,7 @@ stages: - template: 'azure-pipeline-templates/build.yml' # Block Blob Test - - template: 'e2e-tests-spcl.yml' + - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' parameters: conf_template: azure_key.yaml config_file: $(BLOBFUSE2_CFG) @@ -634,7 +634,7 @@ stages: clone: false # ADLS Test - - template: 'e2e-tests-spcl.yml' + - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' parameters: conf_template: azure_key.yaml config_file: $(BLOBFUSE2_CFG) From fccb0310e13eba0a198986b84a8889d1faae0016 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 08:18:48 +0000 Subject: [PATCH 55/87] Changes for nightly pipeline --- azure-pipeline-templates/container.yml | 6 ++++-- blobfuse2-nightly.yaml | 6 +----- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index 9ba6a3b31..02318a699 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -25,11 +25,12 @@ parameters: steps: # Generate the Container Name - ${{ if eq(parameters.generate_container, true) }}: - - bash: | + - script: | CONTAINER_NAME=$(cat /dev/urandom | tr -dc 'a-z0-9' | fold -w 16 | head -n 1) echo "Generated container name: $CONTAINER_NAME" echo "##vso[task.setvariable variable=containerName]$CONTAINER_NAME" displayName: 'Generate random container name' + condition: eq(variables['distro'], 'ubuntu') # Creating the Container - ${{ if eq(parameters.create_container, true) }}: - script: | @@ -40,6 +41,7 @@ steps: --account-key ${{ parameters.account_key }} \ --fail-on-exist displayName: 'Creating ${{ parameters.account_type }} Container' + condition: eq(variables['distro'], 'ubuntu') # Destroying the Container - ${{ if eq(parameters.delete_container, true) }}: @@ -50,4 +52,4 @@ steps: --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ displayName: 'Deleting ${{ parameters.account_type }} Container' - condition: always() \ No newline at end of file + condition: eq(variables['distro'], 'ubuntu') \ No newline at end of file diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 0f92dcfac..7ec0592a2 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -77,7 +77,6 @@ stages: distro: 'ubuntu' distro_version: 'ubuntu_22_arm64' poolName: 'blobfuse-ubn-arm64-pool' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM) fuselib: 'libfuse3-dev' tags: 'fuse3' @@ -169,7 +168,6 @@ stages: distro: 'ubuntu' distro_version: 'ubuntu_20' poolName: 'blobfuse-ubuntu-pool' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22-ADLS: @@ -177,7 +175,6 @@ stages: distro: 'ubuntu' distro_version: 'ubuntu_22_x86' poolName: 'blobfuse-ubuntu-pool' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' Ubuntu-22-ARM64-ADLS: @@ -185,7 +182,6 @@ stages: distro: 'ubuntu' distro_version: 'ubuntu_22_arm64' poolName: 'blobfuse-ubn-arm64-pool' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22_ARM) fuselib: 'libfuse3-dev' tags: 'fuse3' @@ -241,7 +237,7 @@ stages: adls: true account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) - account_sas: $(adlsSas) + account_sas: $(AZTEST_ADLS_SAS) client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) From dbc3a9ddf381af89f4628614b298d972620a82b4 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 10:47:51 +0000 Subject: [PATCH 56/87] Add data consistency checks --- azure-pipeline-templates/data-consistency.yml | 112 ++++++++++++++++++ azure-pipeline-templates/data.yml | 63 ++++++++++ .../e2e-tests-block-cache-data-integrity.yml | 1 - 3 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 azure-pipeline-templates/data-consistency.yml create mode 100644 azure-pipeline-templates/data.yml diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml new file mode 100644 index 000000000..8eac72078 --- /dev/null +++ b/azure-pipeline-templates/data-consistency.yml @@ -0,0 +1,112 @@ +# Data Consistency check across blobfuse different configuration files. +parameters: + - name: config_file + type: string + - name: account_name + type: string + - name: account_type + type: string + - name: account_key + type: string + - name: verbose_log + type: boolean + default: false + - name: idstring + type: string + default: + +steps: + # Generate Data in the local filesystem, This step is only required once at the step + - template: 'data.yml' + parameters: + generate_data: true + # Test data consistency for various Config options. + + # Generate config file for file cache + - script: | + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} + cat ${{ parameters.config_file }} + displayName: 'Create Config File for File Cache' + env: + NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} + NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} + ACCOUNT_TYPE: ${{ parameters.account_type }} + VERBOSE_LOG: ${{ parameters.verbose_log }} + continueOnError: false + + # Mount in File cache + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.idstring }}-filecache + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 + - template: 'data.yml' + parameters: + copy_data: true + check_consistency: true + + # Mount in File cache with direct-io + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.idstring }}-filecache-directio + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 -o direct_io + - template: 'data.yml' + parameters: + copy_data: true + check_consistency: true + + # Generate Config file for block cache + - script: | + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} + cat ${{ parameters.config_file }} + displayName: 'Create Config File for Block Cache' + env: + NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} + NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} + ACCOUNT_TYPE: ${{ parameters.account_type }} + ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} + VERBOSE_LOG: ${{ parameters.verbose_log }} + + # Mount in Block Cache. + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.idstring }} + ro_mount: true + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) + - template: 'data.yml' + parameters: + copy_data: true + check_consistency: true + + # Mount in Block Cache direct-io + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.idstring }} + ro_mount: true + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o direct_io + - template: 'data.yml' + parameters: + copy_data: true + check_consistency: true + + # Mount in Block Cache read-only + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.idstring }} + ro_mount: true + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro + - template: 'data.yml' + parameters: + copy_data: false + check_consistency: true + + \ No newline at end of file diff --git a/azure-pipeline-templates/data.yml b/azure-pipeline-templates/data.yml new file mode 100644 index 000000000..9d1a036c3 --- /dev/null +++ b/azure-pipeline-templates/data.yml @@ -0,0 +1,63 @@ +# Generate files in Local Filesystem. files would be created at $(ROOT_DIR)/data_files/ +# Copy those files using to the mountpoint using CLI utilities. +parameters: + - name: generate_data + type: boolean + default: false + - name: copy_data + type: boolean + default: false + - name: check_consistency + type: boolean + default: false + +steps: + - ${{ if eq(parameters.generate_data, true) }}: + - script: | + DATA_DIR=$(ROOT_DIR)/data_files + sudo rm -rf $(ROOT_DIR) + sudo mkdir -p $(ROOT_DIR) + sudo chown -R `whoami` $(ROOT_DIR) + chmod 777 $(ROOT_DIR) + echo "##vso[task.setvariable variable=DATA_DIR]$DATA_DIR" + displayName: "Create directory structure for storing files in local filesystem" + + - bash: | + generate_random_file() { + local bs=$1 + + for count in "${file_counts[@]}"; do + file_name="$(DATA_DIR)/random_file_${size}_${bs}_${count}.txt" + + echo "Generating file: $file_name" + + # Use dd to create a file with random content + dd if=/dev/urandom of="$file_name" bs="$bs" count="$count" status=progress + + echo "File $file_name generated successfully!" + done + } + file_counts=(1 2 4 7 8 9 10 20 30 50 100 200 1024 2048 4096) + echo "*************************Generate very small files*****************************" + generate_random_file "1" + echo "*************************Generating small files********************************" + generate_random_file "1K" + echo "*************************Generating medium files********************************" + generate_random_file "1M" + displayName: "Generate files with random data using dd command" + + - bash: | + md5sum * > checklist.chk + displayName: "Generate md5sum of the generated files" + workingDirectory: $(DATA_DIR) + + - ${{ if eq(parameters.copy_data, true) }}: + - script: | + cp -v $(DATA_DIR)/* $(MOUNT_DIR)/ + displayName: "Copy Files to the Mountpoint" + + - ${{ if eq(parameters.check_consistency, true) }}: + - script: | + md5sum -c $(DATA_DIR)/checklist.chk + displayName: "Compare the MD5sum of the Mount point with Local filesystem" + workingDirectory: $(MOUNT_DIR) \ No newline at end of file diff --git a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml index 32bfa74bf..724273aa1 100644 --- a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml +++ b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml @@ -19,7 +19,6 @@ parameters: type: string - name: account_type type: string - - name: account_endpoint - name: distro_name type: string - name: quick_test From 50eeff5a46dd73ea702acace7f1c15054e81e85a Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 10:57:57 +0000 Subject: [PATCH 57/87] Fix typo --- .../e2e-tests-block-cache-data-integrity.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml index 724273aa1..32bfa74bf 100644 --- a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml +++ b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml @@ -19,6 +19,7 @@ parameters: type: string - name: account_type type: string + - name: account_endpoint - name: distro_name type: string - name: quick_test From 0a05dcda641e61891099827d79a7d4ea737b836a Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 11:02:59 +0000 Subject: [PATCH 58/87] New staget is not getting recognised by pipeline --- blobfuse2-nightly.yaml | 192 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 171 insertions(+), 21 deletions(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 7ec0592a2..3662bba2c 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -651,7 +651,7 @@ stages: delete_containers: true - - stage: BlockCacheDataValidation + - stage: DataConsistency jobs: # Ubuntu Tests - job: Set_1 @@ -662,14 +662,12 @@ stages: AgentName: 'blobfuse-ubuntu22' distro: 'ubuntu' distro_version: 'ubuntu_22_x86' - containerName: 'test-cnt-ubn-22' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) fuselib: 'libfuse3-dev' tags: 'fuse3' Ubuntu-20: AgentName: 'blobfuse-ubuntu20' - containerName: 'test-cnt-ubn-20' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) + distro: 'ubuntu' + distro_version: 'ubuntu_20_x86' fuselib: 'libfuse-dev' tags: 'fuse2' @@ -696,37 +694,189 @@ stages: value: '$(System.DefaultWorkingDirectory)' - name: WORK_DIR value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: + + steps: # ------------------------------------------------------- - # Pull and build the code + # Pull and build the code and create the containers - template: 'azure-pipeline-templates/build.yml' parameters: skip_ut: true + # Check data consistency for block accounts + - template: 'azure-pipeline-templates/data-consistency.yml' + parameters: + config_file: $(BLOBFUSE2_CFG) + idstring: Block_Blob + account_name: $(AZTEST_BLOCK_ACC_NAME) + account_key: $(AZTEST_BLOCK_KEY) + account_type: block + verbose_log: ${{ parameters.verbose_log }} - - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' + # Check data consistency for adls accounts + - template: 'azure-pipeline-templates/data-consistency.yml' + parameters: + config_file: $(BLOBFUSE2_CFG) + idstring: adls + account_name: $(AZTEST_ADLS_ACC_NAME) + account_key: $(AZTEST_ADLS_KEY) + account_type: adls + verbose_log: ${{ parameters.verbose_log }} + + - template: azure-pipeline-templates/cleanup.yml + parameters: + unmount: true + delete_containers: true + + + + - stage: BlockCacheDataValidation + jobs: + # Ubuntu Tests + - job: Set_1 + timeoutInMinutes: 300 + strategy: + matrix: + Ubuntu-22: + AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' + fuselib: 'libfuse3-dev' + tags: 'fuse3' + Ubuntu-20: + AgentName: 'blobfuse-ubuntu20' + distro: 'ubuntu' + distro_version: 'ubuntu_20_x86' + fuselib: 'libfuse-dev' + tags: 'fuse2' + + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: NightlyBlobFuse + - name: MOUNT_DIR + value: '$(Pipeline.Workspace)/blob_mnt' + - name: TEMP_DIR + value: '$(Pipeline.Workspace)/blobfuse2_tmp' + - name: BLOBFUSE2_CFG + value: '$(Pipeline.Workspace)/blobfuse2.yaml' + - name: BLOBFUSE2_ADLS_CFG + value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + - name: skipComponentGovernanceDetection + value: true + - name: GOPATH + value: '$(Pipeline.Workspace)/go' + - name: ROOT_DIR + value: '$(System.DefaultWorkingDirectory)' + - name: WORK_DIR + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + steps: + # ------------------------------------------------------- + # Pull and build the code and create the containers + - template: 'azure-pipeline-templates/build.yml' + parameters: + skip_ut: true + # Check data consistency for block accounts + - template: 'azure-pipeline-templates/data-consistency.yml' parameters: - conf_template: azure_key.yaml config_file: $(BLOBFUSE2_CFG) - container: $(containerName) idstring: Block_Blob - adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) + account_name: $(AZTEST_BLOCK_ACC_NAME) + account_key: $(AZTEST_BLOCK_KEY) account_type: block - account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - distro_name: $(AgentName) - quick_test: false verbose_log: ${{ parameters.verbose_log }} - clone: true - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) + + # Check data consistency for adls accounts + - template: 'azure-pipeline-templates/data-consistency.yml' + parameters: + config_file: $(BLOBFUSE2_CFG) + idstring: adls + account_name: $(AZTEST_ADLS_ACC_NAME) + account_key: $(AZTEST_ADLS_KEY) + account_type: adls + verbose_log: ${{ parameters.verbose_log }} - template: azure-pipeline-templates/cleanup.yml parameters: unmount: true delete_containers: true + # jobs: + # # Ubuntu Tests + # - job: Set_1 + # timeoutInMinutes: 300 + # strategy: + # matrix: + # Ubuntu-22: + # AgentName: 'blobfuse-ubuntu22' + # distro: 'ubuntu' + # distro_version: 'ubuntu_22_x86' + # containerName: 'test-cnt-ubn-22' + # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) + # fuselib: 'libfuse3-dev' + # tags: 'fuse3' + # Ubuntu-20: + # AgentName: 'blobfuse-ubuntu20' + # containerName: 'test-cnt-ubn-20' + # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) + # fuselib: 'libfuse-dev' + # tags: 'fuse2' + + # pool: + # name: "blobfuse-ubuntu-pool" + # demands: + # - ImageOverride -equals $(AgentName) + + # variables: + # - group: NightlyBlobFuse + # - name: MOUNT_DIR + # value: '$(Pipeline.Workspace)/blob_mnt' + # - name: TEMP_DIR + # value: '$(Pipeline.Workspace)/blobfuse2_tmp' + # - name: BLOBFUSE2_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.yaml' + # - name: BLOBFUSE2_ADLS_CFG + # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + # - name: skipComponentGovernanceDetection + # value: true + # - name: GOPATH + # value: '$(Pipeline.Workspace)/go' + # - name: ROOT_DIR + # value: '$(System.DefaultWorkingDirectory)' + # - name: WORK_DIR + # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + # steps: + # # ------------------------------------------------------- + # # Pull and build the code and create the containers + # - template: 'azure-pipeline-templates/build.yml' + # parameters: + # skip_ut: true + + # - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' + # parameters: + # conf_template: azure_key.yaml + # config_file: $(BLOBFUSE2_CFG) + # container: $(containerName) + # idstring: Block_Blob + # adls: false + # account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) + # account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) + # account_type: block + # account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net + # distro_name: $(AgentName) + # quick_test: false + # verbose_log: ${{ parameters.verbose_log }} + # clone: true + # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable + # temp_dir: $(TEMP_DIR) + # mount_dir: $(MOUNT_DIR) + + # - template: azure-pipeline-templates/cleanup.yml + # parameters: + # unmount: true + # delete_containers: true - stage: BlockCacheDataDirectIOValidation jobs: From 5ca526470478571c0058ee1b5b4756c189cf872b Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 11:05:40 +0000 Subject: [PATCH 59/87] fix typo --- azure-pipeline-templates/data-consistency.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml index 8eac72078..6acf1424d 100644 --- a/azure-pipeline-templates/data-consistency.yml +++ b/azure-pipeline-templates/data-consistency.yml @@ -60,14 +60,13 @@ steps: # Generate Config file for block cache - script: | - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=$(containerName) --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} cat ${{ parameters.config_file }} displayName: 'Create Config File for Block Cache' env: NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} VERBOSE_LOG: ${{ parameters.verbose_log }} # Mount in Block Cache. @@ -77,7 +76,7 @@ steps: ro_mount: true mountStep: script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) - template: 'data.yml' parameters: copy_data: true @@ -90,7 +89,7 @@ steps: ro_mount: true mountStep: script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o direct_io + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o direct_io - template: 'data.yml' parameters: copy_data: true @@ -103,7 +102,7 @@ steps: ro_mount: true mountStep: script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro - template: 'data.yml' parameters: copy_data: false From 697b012e9705aeeeaeb4b215ae9f831c43f8f8c4 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 11:06:38 +0000 Subject: [PATCH 60/87] fix typo --- azure-pipeline-templates/data-consistency.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml index 6acf1424d..4746d99fe 100644 --- a/azure-pipeline-templates/data-consistency.yml +++ b/azure-pipeline-templates/data-consistency.yml @@ -60,7 +60,7 @@ steps: # Generate Config file for block cache - script: | - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=$(containerName) --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} cat ${{ parameters.config_file }} displayName: 'Create Config File for Block Cache' env: From ac80793e121e368d0c7dea46ece6211b0e844577 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 15:05:48 +0000 Subject: [PATCH 61/87] fix typo --- azure-pipeline-templates/data.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/azure-pipeline-templates/data.yml b/azure-pipeline-templates/data.yml index 9d1a036c3..1c024dd8b 100644 --- a/azure-pipeline-templates/data.yml +++ b/azure-pipeline-templates/data.yml @@ -15,10 +15,10 @@ steps: - ${{ if eq(parameters.generate_data, true) }}: - script: | DATA_DIR=$(ROOT_DIR)/data_files - sudo rm -rf $(ROOT_DIR) - sudo mkdir -p $(ROOT_DIR) - sudo chown -R `whoami` $(ROOT_DIR) - chmod 777 $(ROOT_DIR) + sudo rm -rf $DATA_DIR + sudo mkdir -p $DATA_DIR + sudo chown -R `whoami` $DATA_DIR + chmod 777 $DATA_DIR echo "##vso[task.setvariable variable=DATA_DIR]$DATA_DIR" displayName: "Create directory structure for storing files in local filesystem" From 3ef004ae990835ea8047438a38a50372aae648a3 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Mon, 17 Feb 2025 16:42:19 +0000 Subject: [PATCH 62/87] Remove using generic terms --- azure-pipeline-templates/data-consistency.yml | 66 ++- azure-pipeline-templates/data.yml | 4 +- .../e2e-tests-block-cache-data-integrity.yml | 206 --------- .../e2e-tests-block-cache.yml | 256 ----------- blobfuse2-nightly.yaml | 417 +----------------- testdata/scripts/generate-parquet-files.py | 15 - 6 files changed, 65 insertions(+), 899 deletions(-) delete mode 100644 azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml delete mode 100644 azure-pipeline-templates/e2e-tests-block-cache.yml delete mode 100644 testdata/scripts/generate-parquet-files.py diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml index 4746d99fe..cb177d606 100644 --- a/azure-pipeline-templates/data-consistency.yml +++ b/azure-pipeline-templates/data-consistency.yml @@ -37,7 +37,7 @@ steps: # Mount in File cache - template: 'mount.yml' parameters: - prefix: ${{ parameters.idstring }}-filecache + prefix: ${{ parameters.idstring }}-FILECACHE mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 @@ -49,7 +49,7 @@ steps: # Mount in File cache with direct-io - template: 'mount.yml' parameters: - prefix: ${{ parameters.idstring }}-filecache-directio + prefix: ${{ parameters.idstring }}-FILECACHE-DIRECTIO mountStep: script: | $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 -o direct_io @@ -72,7 +72,7 @@ steps: # Mount in Block Cache. - template: 'mount.yml' parameters: - prefix: ${{ parameters.idstring }} + prefix: ${{ parameters.idstring }}-BLOCKCACHE ro_mount: true mountStep: script: | @@ -85,7 +85,7 @@ steps: # Mount in Block Cache direct-io - template: 'mount.yml' parameters: - prefix: ${{ parameters.idstring }} + prefix: ${{ parameters.idstring }}-BLOCKCACHE-DIRECTIO ro_mount: true mountStep: script: | @@ -98,7 +98,7 @@ steps: # Mount in Block Cache read-only - template: 'mount.yml' parameters: - prefix: ${{ parameters.idstring }} + prefix: ${{ parameters.idstring }}-BLOCKCACHE-RDONLY ro_mount: true mountStep: script: | @@ -107,5 +107,59 @@ steps: parameters: copy_data: false check_consistency: true + + # Mount in Block Cache with Disk path + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.idstring }}-BLOCKCACHE-RO-DIO-TEMPPATH + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro -o direct_io --block-cache-path block_cache --block-cache-block-size 8 + - template: 'data.yml' + parameters: + copy_data: true + check_consistency: true + # Now maybe some of the blocks got cache inside the disk + - template: 'data.yml' + parameters: + copy_data: false + check_consistency: true + + # Create Config file for stream + # Context: historically blobfuse has something known as stream as component which was similar to block cache + # It got deprecated and replaced by block cache. + # for backward compatibility the stream configs are redirected to the block cache configs internally. + - script: | + $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_stream.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} + cat ${{ parameters.config_file }} + displayName: 'Create Config File for Block Cache' + env: + NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} + NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} + ACCOUNT_TYPE: ${{ parameters.account_type }} + VERBOSE_LOG: ${{ parameters.verbose_log }} + + # Mount using stream config. + - template: 'mount.yml' + parameters: + prefix: ${{ parameters.idstring }}-STREAM + ro_mount: true + mountStep: + script: | + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) + - template: 'data.yml' + parameters: + copy_data: true + check_consistency: true + + - task: PublishBuildArtifacts@1 + inputs: + pathToPublish: blobfuse2-logs.txt + artifactName: 'blobfuse_block_cache.txt' + condition: failed() + + - script: | + tail -n 200 blobfuse2-logs.txt + displayName: 'View Logs' + condition: failed() - \ No newline at end of file diff --git a/azure-pipeline-templates/data.yml b/azure-pipeline-templates/data.yml index 1c024dd8b..e2bef33b1 100644 --- a/azure-pipeline-templates/data.yml +++ b/azure-pipeline-templates/data.yml @@ -47,7 +47,7 @@ steps: displayName: "Generate files with random data using dd command" - bash: | - md5sum * > checklist.chk + md5sum * | tee checklist.chk displayName: "Generate md5sum of the generated files" workingDirectory: $(DATA_DIR) @@ -56,8 +56,10 @@ steps: cp -v $(DATA_DIR)/* $(MOUNT_DIR)/ displayName: "Copy Files to the Mountpoint" + # drop kernel page cache before getting the md5sums as copy might happened just now - ${{ if eq(parameters.check_consistency, true) }}: - script: | + sudo sh -c "sync; echo 3 > /proc/sys/vm/drop_caches" md5sum -c $(DATA_DIR)/checklist.chk displayName: "Compare the MD5sum of the Mount point with Local filesystem" workingDirectory: $(MOUNT_DIR) \ No newline at end of file diff --git a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml b/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml deleted file mode 100644 index 32bfa74bf..000000000 --- a/azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml +++ /dev/null @@ -1,206 +0,0 @@ -parameters: - - name: conf_template - type: string - - name: config_file - type: string - - name: container - type: string - - name: temp_dir - type: string - - name: mount_dir - type: string - - name: idstring - type: string - - name: adls - type: boolean - - name: account_name - type: string - - name: account_key - type: string - - name: account_type - type: string - - name: account_endpoint - - name: distro_name - type: string - - name: quick_test - type: boolean - default: true - - name: verbose_log - type: boolean - default: false - - name: clone - type: boolean - default: false - - name: block_size_mb - type: string - default: "8" - -steps: - - script: | - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} - displayName: 'Create Config File for RW mount' - env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} - VERBOSE_LOG: ${{ parameters.verbose_log }} - continueOnError: false - - - script: | - cat ${{ parameters.config_file }} - displayName: 'Print config file' - - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 - - - script: | - for i in $(seq 1 10); do echo $(shuf -i 0-4294967296 -n 1); done | parallel --will-cite -j 5 'head -c {} < /dev/urandom > ${{ parameters.mount_dir }}/datafiles_{}' - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > ${{ parameters.mount_dir }}/mixedfiles_{}.txt' - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > ${{ parameters.mount_dir }}/mixedfiles_{}.png' - cd ${{ parameters.mount_dir }} - python3 $(WORK_DIR)/testdata/scripts/generate-parquet-files.py - ls -l ${{ parameters.mount_dir }}/mixedfiles_* - ls -l ${{ parameters.mount_dir }}/datafiles_* - displayName: 'Generate data with File-Cache' - - - script: | - md5sum ${{ parameters.mount_dir }}/datafiles_* > $(WORK_DIR)/md5sum_original_files.txt - md5sum ${{ parameters.mount_dir }}/mixedfiles_* >> $(WORK_DIR)/md5sum_original_files.txt - displayName: 'Generate md5Sum with File-Cache' - - - script: | - echo "----------------------------------------------" - ls -l ${{ parameters.mount_dir }} - displayName: 'Print contents of File-Cache' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RW mount' - - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} - displayName: 'Create Config File for RO mount' - env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} - VERBOSE_LOG: ${{ parameters.verbose_log }} - continueOnError: false - - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - ro_mount: true - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro --block-cache-block-size ${{ parameters.block_size_mb }} - - - script: | - echo "----------------------------------------------" - ls -l ${{ parameters.mount_dir }}/datafiles* - ls -l ${{ parameters.mount_dir }}/mixedfiles* - displayName: 'Print contents of Block-Cache' - - - script: | - md5sum ${{ parameters.mount_dir }}/datafiles_* > $(WORK_DIR)/md5sum_block_cache.txt - md5sum ${{ parameters.mount_dir }}/mixedfiles_* >> $(WORK_DIR)/md5sum_block_cache.txt - displayName: 'Generate md5Sum with Block-Cache' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RW mount' - - - script: | - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_original_files.txt - cat $(WORK_DIR)/md5sum_original_files.txt | cut -d " " -f1 > $(WORK_DIR)/temp.txt && mv $(WORK_DIR)/temp.txt $(WORK_DIR)/md5sum_original_files.txt - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_block_cache.txt - cat $(WORK_DIR)/md5sum_block_cache.txt | cut -d " " -f1 > $(WORK_DIR)/temp.txt && mv $(WORK_DIR)/temp.txt $(WORK_DIR)/md5sum_block_cache.txt - echo "----------------------------------------------" - diff $(WORK_DIR)/md5sum_original_files.txt $(WORK_DIR)/md5sum_block_cache.txt - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Compare md5Sum' - - - script: | - cd $(WORK_DIR) - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key_bc.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} - displayName: 'Create Config File for RO mount with direct-io and disk-cache enabled' - env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} - VERBOSE_LOG: ${{ parameters.verbose_log }} - continueOnError: false - - - script: | - if [ -d "block_cache" ]; then ls -l block_cache; rm -rf block_cache; mkdir block_cache  ; fi - displayName: 'Clear Temp Cache for Block Cache before mounting' - - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - ro_mount: true - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro -o direct_io --block-cache-path block_cache --block-cache-block-size ${{ parameters.block_size_mb }} - - - script: | - echo "----------------------------------------------" - ls -l ${{ parameters.mount_dir }} - displayName: 'Print contents of Block-Cache' - - - script: | - md5sum ${{ parameters.mount_dir }}/datafiles_* > $(WORK_DIR)/md5sum_block_cache_direct_io.txt - md5sum ${{ parameters.mount_dir }}/mixedfiles_* >> $(WORK_DIR)/md5sum_block_cache_direct_io.txt - displayName: 'Generate md5Sum with Block-Cache Direct-IO' - - - script: | - md5sum ${{ parameters.mount_dir }}/datafiles_* > $(WORK_DIR)/md5sum_block_cache_disk_cache.txt - md5sum ${{ parameters.mount_dir }}/mixedfiles_* >> $(WORK_DIR)/md5sum_block_cache_disk_cache.txt - displayName: 'Generate md5Sum with Block-Cache Disk-Cache' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RW mount' - - - script: | - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_original_files.txt - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_block_cache_direct_io.txt | cut -d " " -f1 > $(WORK_DIR)/temp.txt && mv $(WORK_DIR)/temp.txt $(WORK_DIR)/md5sum_block_cache_direct_io.txt - cat $(WORK_DIR)/md5sum_block_cache_direct_io.txt - echo "----------------------------------------------" - diff $(WORK_DIR)/md5sum_original_files.txt $(WORK_DIR)/md5sum_block_cache_direct_io.txt - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Compare md5Sum with Block-Cache Direct-IO' - - - script: | - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_original_files.txt - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_block_cache_disk_cache.txt | cut -d " " -f1 > $(WORK_DIR)/temp.txt && mv $(WORK_DIR)/temp.txt $(WORK_DIR)/md5sum_block_cache_disk_cache.txt - cat $(WORK_DIR)/md5sum_block_cache_disk_cache.txt - echo "----------------------------------------------" - diff $(WORK_DIR)/md5sum_original_files.txt $(WORK_DIR)/md5sum_block_cache_disk_cache.txt - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Compare md5Sum with Block-Cache Disk-Cache' - - - template: 'cleanup.yml' - parameters: - unmount: true - delete_containers: false \ No newline at end of file diff --git a/azure-pipeline-templates/e2e-tests-block-cache.yml b/azure-pipeline-templates/e2e-tests-block-cache.yml deleted file mode 100644 index 592c27d8d..000000000 --- a/azure-pipeline-templates/e2e-tests-block-cache.yml +++ /dev/null @@ -1,256 +0,0 @@ -parameters: - - name: conf_template - type: string - - name: config_file - type: string - - name: container - type: string - - name: temp_dir - type: string - - name: mount_dir - type: string - - name: idstring - type: string - - name: adls - type: boolean - - name: account_name - type: string - - name: account_key - type: string - - name: account_type - type: string - - name: account_endpoint - - name: distro_name - type: string - - name: quick_test - type: boolean - default: true - - name: mnt_flags - type: string - default: "" - - name: verbose_log - type: boolean - default: false - - name: clone - type: boolean - default: false - -steps: - - script: | - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$(WORK_DIR)/testdata/config/azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} - displayName: 'Create Config File for RW mount' - env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} - VERBOSE_LOG: ${{ parameters.verbose_log }} - continueOnError: false - - - script: - cat ${{ parameters.config_file }} - displayName: 'Print config file' - - # run below step only if direct_io is false - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) --file-cache-timeout=3200 ${{ parameters.mnt_flags }} - - - script: | - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > ${{ parameters.mount_dir }}/myfile_{}' - ls -lh ${{ parameters.mount_dir }}/myfile_* - displayName: 'Generate data' - - - script: | - md5sum ${{ parameters.mount_dir }}/myfile_* > $(WORK_DIR)/md5sum_file_cache.txt - displayName: 'Generate md5Sum with File-Cache' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RW mount' - - - script: | - if [ "${{ parameters.idstring }}" = "Stream" ]; then - CONFIG_FILE=$(WORK_DIR)/testdata/config/azure_stream.yaml - else - CONFIG_FILE=$(WORK_DIR)/testdata/config/azure_key_bc.yaml - fi - $(WORK_DIR)/blobfuse2 gen-test-config --config-file=$CONFIG_FILE --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.config_file }} - displayName: 'Create Config File for RO mount' - env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} - VERBOSE_LOG: ${{ parameters.verbose_log }} - continueOnError: false - - - - script: - cat ${{ parameters.config_file }} - displayName: 'Print block cache config file' - - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - ro_mount: true - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro ${{ parameters.mnt_flags }} - - - script: | - md5sum ${{ parameters.mount_dir }}/myfile_* > $(WORK_DIR)/md5sum_block_cache.txt - displayName: 'Generate md5Sum with Block-Cache' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RO mount' - - - script: | - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_block_cache.txt - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_file_cache.txt - echo "----------------------------------------------" - diff $(WORK_DIR)/md5sum_block_cache.txt $(WORK_DIR)/md5sum_file_cache.txt - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Compare md5Sum' - - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - ro_mount: true - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) ${{ parameters.mnt_flags }} - - - script: | - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'cp ${{ parameters.mount_dir }}/myfile_{} ${{ parameters.mount_dir }}/myfileCopy_{}' - md5sum ${{ parameters.mount_dir }}/myfileCopy_* > $(WORK_DIR)/md5sum_block_cache_write.txt - ls -lh ${{ parameters.mount_dir }}/myfile* - displayName: 'Copy files using block-cache' - - - script: | - rm -rf ${{ parameters.mount_dir }}/myfile* - displayName: 'Clear files using block-cache' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RW mount' - - - script: | - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_block_cache_write.txt - cat $(WORK_DIR)/md5sum_block_cache_write.txt | cut -d " " -f1 > $(WORK_DIR)/md5sum_block_cache_write.txt1 - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_file_cache.txt - cat $(WORK_DIR)/md5sum_file_cache.txt | cut -d " " -f1 > $(WORK_DIR)/md5sum_file_cache.txt1 - echo "----------------------------------------------" - diff $(WORK_DIR)/md5sum_block_cache_write.txt1 $(WORK_DIR)/md5sum_file_cache.txt1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Compare md5Sum' - - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - ro_mount: true - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) ${{ parameters.mnt_flags }} - - - script: | - rm -rf $(WORK_DIR)/localfile* - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'head -c {}M < /dev/urandom > $(WORK_DIR)/localfile{}' - displayName: 'Generate local files' - - - script: | - rm -rf ${{ parameters.mount_dir }}/remotefile* - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'cp $(WORK_DIR)/localfile{} ${{ parameters.mount_dir }}/remotefile{}' - displayName: 'Upload local files' - - - script: | - md5sum $(WORK_DIR)/localfile* > $(WORK_DIR)/md5sum_local_modified.txt - md5sum ${{ parameters.mount_dir }}/remotefile* > $(WORK_DIR)/md5sum_remote_modified.txt - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_local_modified.txt - cat $(WORK_DIR)/md5sum_local_modified.txt | cut -d " " -f1 > $(WORK_DIR)/md5sum_local_modified.txt1 - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_remote_modified.txt - cat $(WORK_DIR)/md5sum_remote_modified.txt | cut -d " " -f1 > $(WORK_DIR)/md5sum_remote_modified.txt1 - echo "----------------------------------------------" - diff $(WORK_DIR)/md5sum_local_modified.txt1 $(WORK_DIR)/md5sum_remote_modified.txt1 - if [ $? -ne 0 ]; then - exit 1 - fi - head -c 13M < /dev/urandom > $(WORK_DIR)/additionaldata.data - displayName: 'Compare MD5 before modification' - - - script: | - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'cat $(WORK_DIR)/additionaldata.data >> $(WORK_DIR)/localfile{}' - ls -lh $(WORK_DIR)/localfile* - displayName: 'Modify local files' - - - script: | - for i in {1,2,3,4,5,6,7,8,9,10,20,30,50,100,200,1024,2048,4096}; do echo $i; done | parallel --will-cite -j 5 'cat $(WORK_DIR)/additionaldata.data >> ${{ parameters.mount_dir }}/remotefile{}' - ls -lh ${{ parameters.mount_dir }}/remotefile* - displayName: 'Modify remote files' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RW mount' - - - template: 'mount.yml' - parameters: - prefix: ${{ parameters.idstring }} - ro_mount: true - mountStep: - script: | - $(WORK_DIR)/blobfuse2 mount ${{ parameters.mount_dir }} --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) ${{ parameters.mnt_flags }} - - - script: | - md5sum $(WORK_DIR)/localfile* > $(WORK_DIR)/md5sum_local_modified.txt - md5sum ${{ parameters.mount_dir }}/remotefile* > $(WORK_DIR)/md5sum_remote_modified.txt - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_local_modified.txt - cat $(WORK_DIR)/md5sum_local_modified.txt | cut -d " " -f1 > $(WORK_DIR)/md5sum_local_modified.txt1 - echo "----------------------------------------------" - cat $(WORK_DIR)/md5sum_remote_modified.txt - cat $(WORK_DIR)/md5sum_remote_modified.txt | cut -d " " -f1 > $(WORK_DIR)/md5sum_remote_modified.txt1 - echo "----------------------------------------------" - diff $(WORK_DIR)/md5sum_local_modified.txt1 $(WORK_DIR)/md5sum_remote_modified.txt1 - if [ $? -ne 0 ]; then - exit 1 - fi - displayName: 'Compare MD5 of modified files' - - - script: | - rm -rf $(WORK_DIR)/localfile* - rm -rf ${{ parameters.mount_dir }}/myfile* - displayName: 'Copy files using block-cache' - - - script: | - $(WORK_DIR)/blobfuse2 unmount all - displayName: 'Unmount RW mount' - - - task: PublishBuildArtifacts@1 - inputs: - pathToPublish: blobfuse2-logs.txt - artifactName: 'blobfuse_block_cache.txt' - condition: failed() - - - script: | - tail -n 200 blobfuse2-logs.txt - displayName: 'View Logs' - condition: failed() - - - template: 'cleanup.yml' - parameters: - unmount: true - delete_containers: false \ No newline at end of file diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 3662bba2c..82191a1dd 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -705,7 +705,7 @@ stages: - template: 'azure-pipeline-templates/data-consistency.yml' parameters: config_file: $(BLOBFUSE2_CFG) - idstring: Block_Blob + idstring: BLOCK account_name: $(AZTEST_BLOCK_ACC_NAME) account_key: $(AZTEST_BLOCK_KEY) account_type: block @@ -715,7 +715,7 @@ stages: - template: 'azure-pipeline-templates/data-consistency.yml' parameters: config_file: $(BLOBFUSE2_CFG) - idstring: adls + idstring: ADLS account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) account_type: adls @@ -725,423 +725,10 @@ stages: parameters: unmount: true delete_containers: true - - - - - stage: BlockCacheDataValidation - jobs: - # Ubuntu Tests - - job: Set_1 - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - distro: 'ubuntu' - distro_version: 'ubuntu_22_x86' - fuselib: 'libfuse3-dev' - tags: 'fuse3' - Ubuntu-20: - AgentName: 'blobfuse-ubuntu20' - distro: 'ubuntu' - distro_version: 'ubuntu_20_x86' - fuselib: 'libfuse-dev' - tags: 'fuse2' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: skipComponentGovernanceDetection - value: true - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code and create the containers - - template: 'azure-pipeline-templates/build.yml' - parameters: - skip_ut: true - # Check data consistency for block accounts - - template: 'azure-pipeline-templates/data-consistency.yml' - parameters: - config_file: $(BLOBFUSE2_CFG) - idstring: Block_Blob - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) - account_type: block - verbose_log: ${{ parameters.verbose_log }} - - # Check data consistency for adls accounts - - template: 'azure-pipeline-templates/data-consistency.yml' - parameters: - config_file: $(BLOBFUSE2_CFG) - idstring: adls - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) - account_type: adls - verbose_log: ${{ parameters.verbose_log }} - - - template: azure-pipeline-templates/cleanup.yml - parameters: - unmount: true - delete_containers: true - # jobs: - # # Ubuntu Tests - # - job: Set_1 - # timeoutInMinutes: 300 - # strategy: - # matrix: - # Ubuntu-22: - # AgentName: 'blobfuse-ubuntu22' - # distro: 'ubuntu' - # distro_version: 'ubuntu_22_x86' - # containerName: 'test-cnt-ubn-22' - # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - # Ubuntu-20: - # AgentName: 'blobfuse-ubuntu20' - # containerName: 'test-cnt-ubn-20' - # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) - # fuselib: 'libfuse-dev' - # tags: 'fuse2' - - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - # - name: MOUNT_DIR - # value: '$(Pipeline.Workspace)/blob_mnt' - # - name: TEMP_DIR - # value: '$(Pipeline.Workspace)/blobfuse2_tmp' - # - name: BLOBFUSE2_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.yaml' - # - name: BLOBFUSE2_ADLS_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - # - name: skipComponentGovernanceDetection - # value: true - # - name: GOPATH - # value: '$(Pipeline.Workspace)/go' - # - name: ROOT_DIR - # value: '$(System.DefaultWorkingDirectory)' - # - name: WORK_DIR - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - # steps: - # # ------------------------------------------------------- - # # Pull and build the code and create the containers - # - template: 'azure-pipeline-templates/build.yml' - # parameters: - # skip_ut: true - - # - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' - # parameters: - # conf_template: azure_key.yaml - # config_file: $(BLOBFUSE2_CFG) - # container: $(containerName) - # idstring: Block_Blob - # adls: false - # account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - # account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - # account_type: block - # account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - # distro_name: $(AgentName) - # quick_test: false - # verbose_log: ${{ parameters.verbose_log }} - # clone: true - # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - # temp_dir: $(TEMP_DIR) - # mount_dir: $(MOUNT_DIR) - - # - template: azure-pipeline-templates/cleanup.yml - # parameters: - # unmount: true - # delete_containers: true - - - stage: BlockCacheDataDirectIOValidation - jobs: - # Ubuntu Tests - - job: Set_1 - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-20: - AgentName: 'blobfuse-ubuntu20' - distro: 'ubuntu' - distro_version: 'ubuntu_20_x86' - containerName: 'test-cnt-ubn-20' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) - fuselib: 'libfuse-dev' - tags: 'fuse2' - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - distro: 'ubuntu' - distro_version: 'ubuntu_22_x86' - containerName: 'test-cnt-ubn-22' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - fuselib: 'libfuse3-dev' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: skipComponentGovernanceDetection - value: true - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - skip_ut: true - - - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' - parameters: - conf_template: azure_key.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: Block_Blob - adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - account_type: block - account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - distro_name: $(AgentName) - quick_test: false - verbose_log: ${{ parameters.verbose_log }} - clone: true - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - mnt_flags: "-o direct_io" - - - template: azure-pipeline-templates/cleanup.yml - parameters: - unmount: true - delete_containers: true - - - - - stage: BlockCacheDataIntegrityValidation - jobs: - # Ubuntu Tests - - job: Set_1 - timeoutInMinutes: 300 - strategy: - matrix: - Ubuntu-20: - AgentName: 'blobfuse-ubuntu20' - distro: 'ubuntu' - distro_version: 'ubuntu_20_x86' - containerName: 'test-cnt-ubn-20' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) - fuselib: 'libfuse-dev' - tags: 'fuse2' - Ubuntu-22: - AgentName: 'blobfuse-ubuntu22' - distro: 'ubuntu' - distro_version: 'ubuntu_22_x86' - containerName: 'test-cnt-ubn-22' - adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - fuselib: 'libfuse3-dev' - tags: 'fuse3' - - pool: - name: "blobfuse-ubuntu-pool" - demands: - - ImageOverride -equals $(AgentName) - - variables: - - group: NightlyBlobFuse - - name: MOUNT_DIR - value: '$(Pipeline.Workspace)/blob_mnt' - - name: TEMP_DIR - value: '$(Pipeline.Workspace)/blobfuse2_tmp' - - name: BLOBFUSE2_CFG - value: '$(Pipeline.Workspace)/blobfuse2.yaml' - - name: BLOBFUSE2_ADLS_CFG - value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - - name: skipComponentGovernanceDetection - value: true - - name: GOPATH - value: '$(Pipeline.Workspace)/go' - - name: ROOT_DIR - value: '$(System.DefaultWorkingDirectory)' - - name: WORK_DIR - value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - steps: - # ------------------------------------------------------- - # Pull and build the code - - template: 'azure-pipeline-templates/build.yml' - parameters: - skip_ut: true - - - script: | - sudo apt-get install python3-setuptools -y - sudo apt install python3-pip -y - sudo pip3 install pandas numpy pyarrow fastparquet - displayName: 'Install Python Packages' - - - template: 'azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml' - parameters: - conf_template: azure_key.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: Block_Blob - adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - account_type: block - account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - distro_name: $(AgentName) - quick_test: false - verbose_log: ${{ parameters.verbose_log }} - clone: true - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - block_size_mb: "1" - - - template: 'azure-pipeline-templates/e2e-tests-block-cache-data-integrity.yml' - parameters: - conf_template: azure_key.yaml - config_file: $(BLOBFUSE2_CFG) - container: $(containerName) - idstring: Block_Blob - adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - account_type: block - account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - distro_name: $(AgentName) - quick_test: false - verbose_log: ${{ parameters.verbose_log }} - clone: true - # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - temp_dir: $(TEMP_DIR) - mount_dir: $(MOUNT_DIR) - block_size_mb: "8" - - - template: azure-pipeline-templates/cleanup.yml - parameters: - unmount: true - delete_containers: true - # Run e2e tests for file cache, stream on hns, fns - # - stage: StreamDataValidation - # jobs: - # # Ubuntu Tests - # - job: Set_1 - # timeoutInMinutes: 300 - # strategy: - # matrix: - # Ubuntu-22: - # AgentName: 'blobfuse-ubuntu22' - # containerName: 'test-cnt-ubn-22' - # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - # - name: MOUNT_DIR - # value: '$(Pipeline.Workspace)/blob_mnt' - # - name: TEMP_DIR - # value: '$(Pipeline.Workspace)/blobfuse2_tmp' - # - name: BLOBFUSE2_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.yaml' - # - name: BLOBFUSE2_ADLS_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - # - name: skipComponentGovernanceDetection - # value: true - # - name: GOPATH - # value: '$(Pipeline.Workspace)/go' - # - name: ROOT_DIR - # value: '$(System.DefaultWorkingDirectory)' - # - name: WORK_DIR - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - # steps: - # # ------------------------------------------------------- - # # Pull and build the code - # - template: 'azure-pipeline-templates/build.yml' - # parameters: - # working_directory: $(WORK_DIR) - # root_dir: $(ROOT_DIR) - # mount_dir: $(MOUNT_DIR) - # temp_dir: $(TEMP_DIR) - # gopath: $(GOPATH) - # container: $(containerName) - # tags: $(tags) - # fuselib: $(fuselib) - # skip_ut: true - - # - template: 'azure-pipeline-templates/e2e-tests-block-cache.yml' - # parameters: - # conf_template: azure_stream.yaml - # config_file: $(BLOBFUSE2_CFG) - # container: $(containerName) - # idstring: Stream - # adls: false - # account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - # account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - # account_type: block - # account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - # distro_name: $(AgentName) - # quick_test: false - # verbose_log: ${{ parameters.verbose_log }} - # clone: true - # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - # temp_dir: $(TEMP_DIR) - # mount_dir: $(MOUNT_DIR) - # - stage: FNSDataValidation # jobs: # # Ubuntu Tests diff --git a/testdata/scripts/generate-parquet-files.py b/testdata/scripts/generate-parquet-files.py deleted file mode 100644 index 39d74312f..000000000 --- a/testdata/scripts/generate-parquet-files.py +++ /dev/null @@ -1,15 +0,0 @@ -import pandas as pd -import numpy as np -import random - -# Function to generate a random number of rows for the DataFrame -def random_row_count(min_rows=10, max_rows=1000): - return random.randint(min_rows, max_rows) - -# Generate 10 Parquet files with varying sizes -for i in range(10): - row_count = random_row_count() - df = pd.DataFrame(np.random.randn(row_count, 4), columns=list('ABCD')) - file_name = f'mixedfiles_{i}.parquet' - df.to_parquet(file_name, index=False) - print(f'Created {file_name} with {row_count} rows') From 436cebb3002d3e55594cb8cdaf7a657f6d470959 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 03:37:04 +0000 Subject: [PATCH 63/87] Remove using generic terms --- azure-pipeline-templates/data-consistency.yml | 4 +- azure-pipeline-templates/package-install.yml | 7 +- blobfuse2-nightly.yaml | 161 +----------------- 3 files changed, 8 insertions(+), 164 deletions(-) diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml index cb177d606..ed72acb43 100644 --- a/azure-pipeline-templates/data-consistency.yml +++ b/azure-pipeline-templates/data-consistency.yml @@ -111,10 +111,10 @@ steps: # Mount in Block Cache with Disk path - template: 'mount.yml' parameters: - prefix: ${{ parameters.idstring }}-BLOCKCACHE-RO-DIO-TEMPPATH + prefix: ${{ parameters.idstring }}-BLOCKCACHE-DIO-TEMPPATH mountStep: script: | - $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o ro -o direct_io --block-cache-path block_cache --block-cache-block-size 8 + $(WORK_DIR)/blobfuse2 mount $(MOUNT_DIR) --config-file=${{ parameters.config_file }} --default-working-dir=$(WORK_DIR) -o direct_io --block-cache-path block_cache --block-cache-block-size 8 - template: 'data.yml' parameters: copy_data: true diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index d27d4c99d..6b5efeffe 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -83,17 +83,20 @@ steps: # SUSE PACKAGE INSTALLATION - script: | - sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse + sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse python3 wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm fusermount3 --v wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm + sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm + sudo zypper install -y azure-cli displayName: 'Libfuse Setup ${{ parameters.distro_version }}' condition: eq(variables['distro'], 'suse') # MARINER PACKAGE INSTALLATION - script: | sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y + sudo tdnf install ca-certificates + sudo tdnf install azure-cli displayName: 'Libfuse Setup ${{ parameters.distro_version }}' condition: eq(variables['distro'], 'mariner') \ No newline at end of file diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 82191a1dd..83a3c07e3 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -724,163 +724,4 @@ stages: - template: azure-pipeline-templates/cleanup.yml parameters: unmount: true - delete_containers: true - -# Run e2e tests for file cache, stream on hns, fns - - - # - stage: FNSDataValidation - # jobs: - # # Ubuntu Tests - # - job: Set_1 - # timeoutInMinutes: 300 - # strategy: - # matrix: - # Ubuntu-20: - # AgentName: 'blobfuse-ubuntu20' - # containerName: 'test-cnt-ubn-20' - # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) - # fuselib: 'libfuse-dev' - # tags: 'fuse2' - # Ubuntu-22: - # AgentName: 'blobfuse-ubuntu22' - # containerName: 'test-cnt-ubn-22' - # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - # - name: MOUNT_DIR - # value: '$(Pipeline.Workspace)/blob_mnt' - # - name: TEMP_DIR - # value: '$(Pipeline.Workspace)/blobfuse2_tmp' - # - name: BLOBFUSE2_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.yaml' - # - name: BLOBFUSE2_ADLS_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - # - name: skipComponentGovernanceDetection - # value: true - # - name: GOPATH - # value: '$(Pipeline.Workspace)/go' - # - name: ROOT_DIR - # value: '$(System.DefaultWorkingDirectory)' - # - name: WORK_DIR - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - # steps: - # # ------------------------------------------------------- - # # Pull and build the code - # - template: 'azure-pipeline-templates/build.yml' - # parameters: - # working_directory: $(WORK_DIR) - # root_dir: $(ROOT_DIR) - # mount_dir: $(MOUNT_DIR) - # temp_dir: $(TEMP_DIR) - # gopath: $(GOPATH) - # container: $(containerName) - # tags: $(tags) - # fuselib: $(fuselib) - # skip_ut: true - - # - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' - # parameters: - # conf_template: azure_key.yaml - # config_file: $(BLOBFUSE2_CFG) - # container: $(containerName) - # idstring: Block_Blob - # adls: false - # account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - # account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - # account_type: block - # account_endpoint: https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net - # distro_name: $(AgentName) - # quick_test: false - # verbose_log: ${{ parameters.verbose_log }} - # clone: true - # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - # temp_dir: $(TEMP_DIR) - # mount_dir: $(MOUNT_DIR) - - # - stage: HNSDataValidation - # jobs: - # # Ubuntu Tests - # - job: Set_1 - # timeoutInMinutes: 300 - # strategy: - # matrix: - # Ubuntu-20: - # AgentName: 'blobfuse-ubuntu20' - # containerName: 'test-cnt-ubn-20' - # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_20) - # fuselib: 'libfuse-dev' - # tags: 'fuse2' - # Ubuntu-22: - # AgentName: 'blobfuse-ubuntu22' - # containerName: 'test-cnt-ubn-22' - # adlsSas: $(AZTEST_ADLS_CONT_SAS_UBN_22) - # fuselib: 'libfuse3-dev' - # tags: 'fuse3' - - # pool: - # name: "blobfuse-ubuntu-pool" - # demands: - # - ImageOverride -equals $(AgentName) - - # variables: - # - group: NightlyBlobFuse - # - name: MOUNT_DIR - # value: '$(Pipeline.Workspace)/blob_mnt' - # - name: TEMP_DIR - # value: '$(Pipeline.Workspace)/blobfuse2_tmp' - # - name: BLOBFUSE2_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.yaml' - # - name: BLOBFUSE2_ADLS_CFG - # value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' - # - name: skipComponentGovernanceDetection - # value: true - # - name: GOPATH - # value: '$(Pipeline.Workspace)/go' - # - name: ROOT_DIR - # value: '$(System.DefaultWorkingDirectory)' - # - name: WORK_DIR - # value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' - - # steps: - # # ------------------------------------------------------- - # # Pull and build the code - # - template: 'azure-pipeline-templates/build.yml' - # parameters: - # working_directory: $(WORK_DIR) - # root_dir: $(ROOT_DIR) - # mount_dir: $(MOUNT_DIR) - # temp_dir: $(TEMP_DIR) - # gopath: $(GOPATH) - # container: $(containerName) - # tags: $(tags) - # fuselib: $(fuselib) - # skip_ut: true - - # - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' - # parameters: - # conf_template: azure_key.yaml - # config_file: $(BLOBFUSE2_CFG) - # container: $(containerName) - # idstring: ADLS - # adls: true - # account_name: $(AZTEST_ADLS_ACC_NAME) - # account_key: $(AZTEST_ADLS_KEY) - # account_type: adls - # account_endpoint: https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net - # distro_name: $(AgentName) - # quick_test: false - # verbose_log: ${{ parameters.verbose_log }} - # clone: true - # # TODO: These can be removed one day and replace all instances of ${{ parameters.temp_dir }} with $(TEMP_DIR) since it is a global variable - # temp_dir: $(TEMP_DIR) - # mount_dir: $(MOUNT_DIR) + delete_containers: true \ No newline at end of file From 4e31fd2232db4be421320810b72750845026f40f Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 04:29:36 +0000 Subject: [PATCH 64/87] revert some files to main --- .github/pull_request_template.md | 30 +++++++++++++--------- LICENSE | 43 +++++++++++++++++++------------- 2 files changed, 44 insertions(+), 29 deletions(-) diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 60d622098..6c403d854 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,18 @@ - - -- [ ] The purpose of this PR is explained in this or a referenced issue. -- [ ] Tests are included and/or updated for code changes. -- [ ] Updates to module CHANGELOG.md are included. -- [ ] License headers are included in each file. -- [ ] Need Documentation change (eg: readme.md). - +## ✅ What + + + +## 🤔 Why + + + +## 👩‍🔬 How to validate if applicable + + + +## 🔖 Related links + +- [Issues]() +- [Team thread]() +- [Documents]() +- [Email Subject] diff --git a/LICENSE b/LICENSE index 72c2b5d92..41eb56af9 100755 --- a/LICENSE +++ b/LICENSE @@ -1,21 +1,30 @@ -The MIT License (MIT) + _____ _____ _____ ____ ______ _____ ------ + | | | | | | | | | | | | | + | | | | | | | | | | | | | + | --- | | | | |-----| |---- | | |-----| |----- ------ + | | | | | | | | | | | | | + | ____| |_____ | ____| | ____| | |_____| _____| |_____ |_____ -Copyright © Microsoft Corporation -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: + Licensed under the MIT License . -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. + Copyright © 2020-2025 Microsoft Corporation. All rights reserved. + Author : -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. \ No newline at end of file + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE From 0de28e9b2767e61aaa3c0879025ff74e1fbfb581 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 04:48:26 +0000 Subject: [PATCH 65/87] modify comments --- azure-pipeline-templates/build.yml | 2 +- azure-pipeline-templates/data.yml | 2 +- blobfuse2-ci.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 3a05e2439..ec3cafc80 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -1,7 +1,7 @@ # This step is responsible for the following behaviour: # 1. Setup the VM all necessary packages. # 2. Build blobfuse, bfusemon, install AzCLI -# 3. Create Temporary Containers in the storage account with choosing random string as name and also sets the global containerName paramter to this value. +# 3. Create Temporary Containers in the storage account with choosing random string as name and also sets the global containerName parameter to this value. # Caution: The Deletion of the container is left to the caller of this file, caller must call the cleanup.yml with the parameter delete_containers: true at the end of the job # 4. Run Unit Tests if specified by parameter skip_ut: false diff --git a/azure-pipeline-templates/data.yml b/azure-pipeline-templates/data.yml index e2bef33b1..56584362c 100644 --- a/azure-pipeline-templates/data.yml +++ b/azure-pipeline-templates/data.yml @@ -1,4 +1,4 @@ -# Generate files in Local Filesystem. files would be created at $(ROOT_DIR)/data_files/ +# Generate files in Local Filesystem. files would be created at $(DATA_DIR) # Copy those files using to the mountpoint using CLI utilities. parameters: - name: generate_data diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 5c69895d8..bd24d7edb 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -7,7 +7,7 @@ pr: - main -# Run CI when changes occured to main branch / branch with name ci/* +# Run CI when changes occurred to main branch / branch with name ci/* trigger: - main - ci/* From a87132a556900ed1b8d3186029d2d03b4bae693e Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 05:22:49 +0000 Subject: [PATCH 66/87] * Add static container names for distros other than ubuntu coz installation seems to be complicated in them. --- azure-pipeline-templates/package-install.yml | 3 --- blobfuse2-nightly.yaml | 9 +++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index 6b5efeffe..c2a4c0470 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -89,14 +89,11 @@ steps: fusermount3 --v wget $(SUSE_FUSE3_PATH)/fuse3-devel-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-devel-3.10.5-150400.1.7.x86_64.rpm - sudo zypper install -y azure-cli displayName: 'Libfuse Setup ${{ parameters.distro_version }}' condition: eq(variables['distro'], 'suse') # MARINER PACKAGE INSTALLATION - script: | sudo tdnf install gcc build-essential git fuse3 fuse3-devel python3 -y - sudo tdnf install ca-certificates - sudo tdnf install azure-cli displayName: 'Libfuse Setup ${{ parameters.distro_version }}' condition: eq(variables['distro'], 'mariner') \ No newline at end of file diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 83a3c07e3..9584b08c1 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -530,6 +530,7 @@ stages: distro_version: 'RHEL-8.6' distro: rhel poolName: 'blobfuse-rhel-pool' + containerName: "test-cnt-rhel-86" Description: 'Red Hat Enterprise Linux 8.6' AgentName: 'blobfuse-rhel8_6' tags: 'fuse3' @@ -537,6 +538,7 @@ stages: distro_version: 'RHEL-9.0' distro: rhel poolName: 'blobfuse-rhel-pool' + containerName: "test-cnt-rhel-9" Description: 'Red Hat Enterprise Linux 9.0' AgentName: 'blobfuse-rhel9' tags: 'fuse3' @@ -544,24 +546,28 @@ stages: distro_version: "CentOS-7.9" distro: centos poolName: 'blobfuse-centos-pool' + containerName: "test-cnt-cent-7" Description: "CentOS 7.9" AgentName: "blobfuse-centos7" CentOS-8.5: distro_version: "CentOS-8.5" distro: centos poolName: 'blobfuse-centos-pool' + containerName: "test-cnt-cent-8" Description: "CentOS 8.5" AgentName: "blobfuse-centos8" Oracle-8.1: distro_version: "Oracle-8.1" distro: oracle poolName: 'blobfuse-oracle-pool' + containerName: "test-cnt-ora-81" Description: "Oracle Linux 8.1" AgentName: "blobfuse-oracle81" Rocky-8.0: distro_version: "Rocky-8.0" distro: rocky poolName: 'blobfuse2-rocky-pool' + containerName: "test-cnt-rocky-8" Description: "Rocky Linux 8.0" AgentName: "blobfuse-rocky8" tags: 'fuse3' @@ -569,6 +575,7 @@ stages: distro_version: "Rocky-9.0" distro: rocky poolName: 'blobfuse2-rocky-pool' + containerName: "test-cnt-rocky-9" Description: "Rocky Linux 9.0" AgentName: "blobfuse-rocky9" tags: 'fuse3' @@ -576,12 +583,14 @@ stages: distro_version: "SUSE-15" distro: suse poolName: 'blobfuse-suse-pool' + containerName: "test-cnt-suse-15" Description: "SUSE Enterprise Linux 15" AgentName: "blobfuse-suse15" Mariner2: distro_version: "Mariner2" distro: mariner poolName: 'blobfuse-mariner-pool' + containerName: "test-cnt-mari-2" Description: "CBL-Mariner2 Linux" AgentName: "blobfuse-mariner2" fuselib: 'libfuse3-dev' From bb90db42859621b3b34308f8fa0327579c4f63a6 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 05:52:59 +0000 Subject: [PATCH 67/87] Stop Checking out the code automatically, lets do it manually always --- azure-pipeline-templates/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index ec3cafc80..5ad3d3be6 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -23,6 +23,7 @@ parameters: default: false steps: + - checkout: none - script: | sudo rm -rf $(ROOT_DIR) sudo mkdir -p $(ROOT_DIR) From 0f2e1b3be1e30dbb47341544cadfd183c33c04d5 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 07:00:41 +0000 Subject: [PATCH 68/87] Use Azcli for only ubuntu --- azure-pipeline-templates/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 5ad3d3be6..aa5969097 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -190,11 +190,11 @@ steps: continueOnError: false workingDirectory: $(WORK_DIR) - # azcli login + # azcli login, This is only for ubuntu distributions, where we create containers for the job - script: | az login --identity --username $(ACLI_BLOBFUSE_MSI_APP_ID) displayName: 'Azure CLI login' - condition: eq('${{ parameters.skip_azcli }}', 'false') + condition: eq(variables['distro'], 'ubuntu') # Running unit tests - task: Go@0 From a1b7f14004fd4472e2aaf0ce70b90598aad732e0 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 11:17:00 +0000 Subject: [PATCH 69/87] Remove Azcli tests for other distros --- azure-pipeline-templates/build.yml | 2 +- blobfuse2-nightly.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index aa5969097..17f98dbad 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -11,7 +11,7 @@ parameters: default: "true" - name: skip_azcli type: string - default: "false" + default: "true" - name: skip_bfusemon type: boolean default: true diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 9584b08c1..78ff278a8 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -622,6 +622,8 @@ stages: steps: # Build the code - template: 'azure-pipeline-templates/build.yml' + parameters: + skip_azcli: "false" # Block Blob Test - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' From 54112dad9893d60ac567ea4f3e85e0250b496fa8 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 17:10:38 +0000 Subject: [PATCH 70/87] Create sas for ADLS container which we create as the account SAS is not sufficient for changing the ACL's --- azure-pipeline-templates/build.yml | 2 -- azure-pipeline-templates/container.yml | 27 ++++++++++++++++++++++---- blobfuse2-nightly.yaml | 14 ++++++++++--- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 17f98dbad..5f5ec7c76 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -151,7 +151,6 @@ steps: account_type: "block" account_name: $(AZTEST_BLOCK_ACC_NAME) account_key: $(AZTEST_BLOCK_KEY) - container_name: $(containerName) # Create a Container in the adls storage account - template: 'container.yml' @@ -160,7 +159,6 @@ steps: account_type: "adls" account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) - container_name: $(containerName) - script: | cnfFile=$HOME/azuretest.json diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index 02318a699..d2ba70a3c 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -9,6 +9,9 @@ parameters: - name: delete_container type: boolean default: false + - name: generate_adls_sas + type: boolean + default: false - name: account_type # accepted values: block, adls type: string default: invalid @@ -35,20 +38,36 @@ steps: - ${{ if eq(parameters.create_container, true) }}: - script: | az --version - echo "Creating Container: ${{ parameters.container_name }}" - az storage container create --name ${{ parameters.container_name }} \ + echo "Creating Container: $(containerName)" + az storage container create --name $(containerName) \ --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ --fail-on-exist displayName: 'Creating ${{ parameters.account_type }} Container' condition: eq(variables['distro'], 'ubuntu') + + # Generate ADLS SAS for the container as account SAS is not enough for modifying ACL's + # set the generated container SAS to the global variable + - ${{ if eq(parameters.generate_adls_sas, true) }}: + - script: | + az --version + end=`date -u -d "70 minutes" '+%Y-%m-%dT%H:%MZ'` + sas=`az storage fs generate-sas -n $(containerName) \ + --account-name ${{ parameters.account_name }} \ + --account-key ${{ parameters.account_key }} \ + --permissions acdelmoprw \ + --expiry $end \ + -o tsv` + echo "##vso[task.setvariable variable=GENERATED_ADLS_CONTAINER_SAS]$sas" + displayName: 'Creating ADLS SAS for ${{ parameters.account_type }} Container' + condition: eq(variables['distro'], 'ubuntu') # Destroying the Container - ${{ if eq(parameters.delete_container, true) }}: - script: | az --version - echo "Deleting Container: ${{ parameters.container_name }}" - az storage container delete --name ${{ parameters.container_name }} \ + echo "Deleting Container: $(containerName)" + az storage container delete --name $(containerName) \ --account-name ${{ parameters.account_name }} \ --account-key ${{ parameters.account_key }} \ displayName: 'Deleting ${{ parameters.account_type }} Container' diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 78ff278a8..24fb1d1ad 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -226,6 +226,16 @@ stages: parameters: skip_ut: true # Skip UT because Block Blob set runs it skip_azcli: "false" + + #----------------------------------------------------------- + # Generate SAS for created container in ADLS with all the permissions. + # Reason: the account SAS is not sufficient to change the ACL's + - template: 'container.yml' + parameters: + generate_adls_sas: true + account_type: "adls" + account_name: $(AZTEST_ADLS_ACC_NAME) + account_key: $(AZTEST_ADLS_KEY) # ------------------------------------------------------- - ${{ if eq(parameters.exhaustive_test, true) }}: @@ -237,7 +247,7 @@ stages: adls: true account_name: $(AZTEST_ADLS_ACC_NAME) account_key: $(AZTEST_ADLS_KEY) - account_sas: $(AZTEST_ADLS_SAS) + account_sas: $(GENERATED_ADLS_CONTAINER_SAS) client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) @@ -622,8 +632,6 @@ stages: steps: # Build the code - template: 'azure-pipeline-templates/build.yml' - parameters: - skip_azcli: "false" # Block Blob Test - template: 'azure-pipeline-templates/e2e-tests-spcl.yml' From 910681499af4b1a641ce51fc0e2221d16283c224 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 18 Feb 2025 17:13:06 +0000 Subject: [PATCH 71/87] typo fix --- blobfuse2-nightly.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 24fb1d1ad..ed750573f 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -230,7 +230,7 @@ stages: #----------------------------------------------------------- # Generate SAS for created container in ADLS with all the permissions. # Reason: the account SAS is not sufficient to change the ACL's - - template: 'container.yml' + - template: 'azure-pipeline-templates/container.yml' parameters: generate_adls_sas: true account_type: "adls" From 0e12d3063ec2c748577cd856b18717b1bf92617b Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 19 Feb 2025 04:27:15 +0000 Subject: [PATCH 72/87] tmp path not getting cleared in block cache --- azure-pipeline-templates/data-consistency.yml | 8 ++++++- azure-pipeline-templates/huge-list-test.yml | 24 +++++++------------ azure-pipeline-templates/verbose-tests.yml | 3 --- 3 files changed, 16 insertions(+), 19 deletions(-) diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml index ed72acb43..897619925 100644 --- a/azure-pipeline-templates/data-consistency.yml +++ b/azure-pipeline-templates/data-consistency.yml @@ -107,7 +107,13 @@ steps: parameters: copy_data: false check_consistency: true - + + # The following script is a bug in block cache, where temp directory given was not getting cleaned after unmount. + # one can remove this when that is solved. + - script: | + if [ -d "block_cache" ]; then ls -l block_cache; rm -rf block_cache; mkdir block_cache  ; fi + displayName: '[BUG]Clear Temp Cache for Block Cache before mounting' + # Mount in Block Cache with Disk path - template: 'mount.yml' parameters: diff --git a/azure-pipeline-templates/huge-list-test.yml b/azure-pipeline-templates/huge-list-test.yml index 2d2a58a2d..5e0b820c6 100755 --- a/azure-pipeline-templates/huge-list-test.yml +++ b/azure-pipeline-templates/huge-list-test.yml @@ -1,10 +1,4 @@ parameters: - - name: working_dir - type: string - - name: mount_dir - type: string - - name: temp_dir - type: string - name: idstring type: string default: '' @@ -14,12 +8,12 @@ parameters: type: string steps: - script: | - sudo fusermount -u ${mount_dir} - sudo fusermount3 -u ${mount_dir} + sudo fusermount -u $(MOUNT_DIR) + sudo fusermount3 -u $(MOUNT_DIR) sudo kill -9 `pidof blobfuse2` || true timeoutInMinutes: 20 env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) displayName: 'HugeList: ${{ parameters.idstring }} PreStart Cleanup' condition: always() @@ -35,18 +29,18 @@ steps: exit $? displayName: 'HugeList: ${{ parameters.idstring }} Verify Mount' env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) # ---------------------------------------- # Huge Directory Listing Testing # Directory Listing Tests - - script: ls -1 ${mount_dir}/huge | wc -l + - script: ls -1 $(MOUNT_DIR)/huge | wc -l timeoutInMinutes: 120 displayName: 'HugeList: ${{ parameters.idstring }} Listing' continueOnError: false env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) - script: grep "OUTGOING REQUEST" blobfuse2-logs.txt | wc -l displayName: 'HugeList: ${{ parameters.idstring }} Request Count' @@ -64,12 +58,12 @@ steps: # Never cleanup here on container otherwise we lose the huge data, just unmount and go - script: | - sudo fusermount -u ${mount_dir} - sudo fusermount3 -u ${mount_dir} + sudo fusermount -u $(MOUNT_DIR) + sudo fusermount3 -u $(MOUNT_DIR) sudo kill -9 `pidof blobfuse2` || true timeoutInMinutes: 5 env: - mount_dir: ${{ parameters.mount_dir }} + mount_dir: $(MOUNT_DIR) displayName: 'HugeList: ${{ parameters.idstring }} Unmount' condition: always() diff --git a/azure-pipeline-templates/verbose-tests.yml b/azure-pipeline-templates/verbose-tests.yml index 2535b815f..6f0d1f6f1 100644 --- a/azure-pipeline-templates/verbose-tests.yml +++ b/azure-pipeline-templates/verbose-tests.yml @@ -326,9 +326,6 @@ steps: - template: huge-list-test.yml parameters: - working_dir: $(WORK_DIR) - mount_dir: $(MOUNT_DIR) - temp_dir: $(TEMP_DIR) idstring: ${{ parameters.service }} Huge List Test distro_name: ${{ parameters.distro_name }} mountStep: From d7cbcbec269dee21310dca9fee2de2a136b675b3 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 19 Feb 2025 07:05:43 +0000 Subject: [PATCH 73/87] Update lvalues --- azure-pipeline-templates/bfusemon.yml | 8 ++-- azure-pipeline-templates/cleanup.yml | 6 +-- azure-pipeline-templates/data-consistency.yml | 18 +++---- azure-pipeline-templates/e2e-tests-spcl.yml | 6 +-- azure-pipeline-templates/package-install.yml | 1 + .../release-distro-tests.yml | 16 +++---- azure-pipeline-templates/verbose-tests.yml | 46 +++++++++--------- blobfuse2-code-coverage.yaml | 48 +++++++++---------- blobfuse2-nightly.yaml | 38 +++++++-------- blobfuse2-perf.yaml | 32 ++++++------- testdata/config/azure_block_perf.yaml | 8 ++-- testdata/config/azure_cli.yaml | 6 +-- testdata/config/azure_key.yaml | 8 ++-- testdata/config/azure_key_bc.yaml | 8 ++-- testdata/config/azure_key_directio.yaml | 8 ++-- testdata/config/azure_key_emptyfile.yaml | 8 ++-- testdata/config/azure_key_hmon.yaml | 8 ++-- testdata/config/azure_key_huge.yaml | 8 ++-- testdata/config/azure_key_lru_purge.yaml | 8 ++-- testdata/config/azure_key_perf.yaml | 8 ++-- testdata/config/azure_key_proxy.yaml | 8 ++-- testdata/config/azure_key_symlink.yaml | 8 ++-- testdata/config/azure_msi.yaml | 8 ++-- testdata/config/azure_msi_sysid.yaml | 4 +- testdata/config/azure_sas.yaml | 8 ++-- testdata/config/azure_sas_proxy.yaml | 6 +-- testdata/config/azure_spn.yaml | 4 +- testdata/config/azure_spn_proxy.yaml | 2 +- testdata/config/azure_stream.yaml | 8 ++-- testdata/config/azure_stream_direct.yaml | 8 ++-- testdata/config/azure_stream_filename.yaml | 7 ++- .../config/azure_stream_filename_direct.yaml | 8 ++-- testdata/config/config_key.yaml | 4 +- testdata/config/config_msi.yaml | 2 +- testdata/config/config_sas.yaml | 4 +- 35 files changed, 193 insertions(+), 193 deletions(-) diff --git a/azure-pipeline-templates/bfusemon.yml b/azure-pipeline-templates/bfusemon.yml index 0bb5b3f68..9764d15f4 100644 --- a/azure-pipeline-templates/bfusemon.yml +++ b/azure-pipeline-templates/bfusemon.yml @@ -4,10 +4,10 @@ steps: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_hmon.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: Create Key Config File env: - ACCOUNT_TYPE: 'block' - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_TYPE: 'block' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false HMON_OUTPUT: $(WORK_DIR) diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index 2f5c3f278..292fc0c53 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -15,11 +15,11 @@ steps: displayName: 'Check process info' - script: | - sudo fusermount -u $(MOUNT_DIR) - sudo fusermount3 -u $(MOUNT_DIR) - sudo kill -9 `pidof blobfuse2` || true + sudo umount -f $(MOUNT_DIR) + sudo kill -9 `pidof blobfuse2` &> /dev/null rm -rf $(MOUNT_DIR)/* rm -rf $(TEMP_DIR)/* + sudo cat /etc/mtab timeoutInMinutes: 5 displayName: 'Unmount Container' condition: always() diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml index 897619925..f7f347058 100644 --- a/azure-pipeline-templates/data-consistency.yml +++ b/azure-pipeline-templates/data-consistency.yml @@ -28,9 +28,9 @@ steps: cat ${{ parameters.config_file }} displayName: 'Create Config File for File Cache' env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.STO_ACC_TYPE }} VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -64,9 +64,9 @@ steps: cat ${{ parameters.config_file }} displayName: 'Create Config File for Block Cache' env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} VERBOSE_LOG: ${{ parameters.verbose_log }} # Mount in Block Cache. @@ -140,9 +140,9 @@ steps: cat ${{ parameters.config_file }} displayName: 'Create Config File for Block Cache' env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.STO_ACC_TYPE }} VERBOSE_LOG: ${{ parameters.verbose_log }} # Mount using stream config. diff --git a/azure-pipeline-templates/e2e-tests-spcl.yml b/azure-pipeline-templates/e2e-tests-spcl.yml index 9fff1b481..cecd79125 100644 --- a/azure-pipeline-templates/e2e-tests-spcl.yml +++ b/azure-pipeline-templates/e2e-tests-spcl.yml @@ -34,9 +34,9 @@ steps: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=${{ parameters.conf_template }} --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config_file }} displayName: 'Create Config File : ${{ parameters.idstring }}' env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false diff --git a/azure-pipeline-templates/package-install.yml b/azure-pipeline-templates/package-install.yml index c2a4c0470..88cb2da9f 100644 --- a/azure-pipeline-templates/package-install.yml +++ b/azure-pipeline-templates/package-install.yml @@ -83,6 +83,7 @@ steps: # SUSE PACKAGE INSTALLATION - script: | + sudo zypper update -y sudo zypper -n install git golang make cmake gcc gcc-c++ glibc-devel fuse python3 wget $(SUSE_FUSE3_PATH)/fuse3-3.10.5-150400.1.7.x86_64.rpm sudo zypper -n --no-gpg-checks install fuse3-3.10.5-150400.1.7.x86_64.rpm diff --git a/azure-pipeline-templates/release-distro-tests.yml b/azure-pipeline-templates/release-distro-tests.yml index c9bad5c6b..c05588472 100644 --- a/azure-pipeline-templates/release-distro-tests.yml +++ b/azure-pipeline-templates/release-distro-tests.yml @@ -33,10 +33,10 @@ steps: blobfuse2 gen-test-config --config-file=${{ parameters.root_dir }}/azure-storage-fuse/testdata/config/azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.root_dir }}/block_blob_config.yaml displayName: 'Create Blob Config File' env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false continueOnError: false @@ -92,10 +92,10 @@ steps: blobfuse2 gen-test-config --config-file=${{ parameters.root_dir }}/azure-storage-fuse/testdata/config/azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.root_dir }}/adls_config.yaml displayName: 'Create ADLS Config File' env: - NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_TYPE: 'adls' + STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: false USE_HTTP: false continueOnError: false diff --git a/azure-pipeline-templates/verbose-tests.yml b/azure-pipeline-templates/verbose-tests.yml index 6f0d1f6f1..700bad360 100644 --- a/azure-pipeline-templates/verbose-tests.yml +++ b/azure-pipeline-templates/verbose-tests.yml @@ -59,10 +59,10 @@ steps: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config }} displayName: Create Key Config File env: - ACCOUNT_TYPE: ${{ parameters.account_type }} - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} + STO_ACC_TYPE: ${{ parameters.account_type }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_ENDPOINT: ${{ parameters.account_endpoint }} VERBOSE_LOG: ${{ parameters.verbose_log }} USE_HTTP: false condition: ${{ parameters.test_key_credential }} @@ -78,10 +78,10 @@ steps: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_sas.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.sas_credential_config }} displayName: Create SAS Config File env: - NIGHTLY_STO_BLOB_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_SAS: ${{ parameters.account_sas }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_SAS: ${{ parameters.account_sas }} + STO_ACC_TYPE: ${{ parameters.account_type }} + STO_ACC_ENDPOINT: ${{ parameters.account_endpoint }} VERBOSE_LOG: ${{ parameters.verbose_log }} condition: ${{ parameters.test_sas_credential }} continueOnError: false @@ -96,9 +96,9 @@ steps: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_cli.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.azcli_credential_config }} displayName: Create Azure CLI Config File env: - NIGHTLY_STO_BLOB_ACC_NAME: ${{ parameters.account_name }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_TYPE: ${{ parameters.account_type }} + STO_ACC_ENDPOINT: ${{ parameters.account_endpoint }} VERBOSE_LOG: ${{ parameters.verbose_log }} condition: ${{ parameters.test_azcli_credential }} continueOnError: false @@ -114,10 +114,10 @@ steps: displayName: Create Key Config File # Note: It is fine to hardcode these credentials since they are publicly documented fake credentials for Azurite env: - ACCOUNT_TYPE: ${{ parameters.account_type }} - NIGHTLY_STO_ACC_NAME: 'devstoreaccount1' - NIGHTLY_STO_ACC_KEY: 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==' - ACCOUNT_ENDPOINT: 'http://127.0.0.1:10000/devstoreaccount1' + STO_ACC_TYPE: ${{ parameters.account_type }} + STO_ACC_NAME: 'devstoreaccount1' + STO_ACC_KEY: 'Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==' + STO_ACC_ENDPOINT: 'http://127.0.0.1:10000/devstoreaccount1' USE_HTTP: 'true' VERBOSE_LOG: ${{ parameters.verbose_log }} condition: ${{ parameters.test_azurite }} @@ -281,10 +281,10 @@ steps: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config }} displayName: Create Config File env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} + STO_ACC_ENDPOINT: ${{ parameters.account_endpoint }} VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -314,10 +314,10 @@ steps: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_huge.yaml --container-name=${{ parameters.huge_container }} --temp-path=$(TEMP_DIR) --output-file=${{ parameters.config }} displayName: Create Config File env: - NIGHTLY_STO_ACC_NAME: ${{ parameters.account_name }} - NIGHTLY_STO_ACC_KEY: ${{ parameters.account_key }} - ACCOUNT_TYPE: ${{ parameters.account_type }} - ACCOUNT_ENDPOINT: ${{ parameters.account_endpoint }} + STO_ACC_NAME: ${{ parameters.account_name }} + STO_ACC_KEY: ${{ parameters.account_key }} + STO_ACC_TYPE: ${{ parameters.account_type }} + STO_ACC_ENDPOINT: ${{ parameters.account_endpoint }} VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 96f99bc41..cc69048ad 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -71,10 +71,10 @@ stages: cd $(WORK_DIR) ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_gentest1.cov gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - Block Blob' @@ -128,10 +128,10 @@ stages: cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) env: - NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_TYPE: 'adls' + STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - ADLS' @@ -186,10 +186,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: 'Create Config File - Block Blob' env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false @@ -408,10 +408,10 @@ stages: workingDirectory: $(WORK_DIR) displayName: "CLI : Secure Config" env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false @@ -478,10 +478,10 @@ stages: cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - Block Blob Proxy' @@ -509,10 +509,10 @@ stages: cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) env: - NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_TYPE: 'adls' + STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - ADLS Proxy' diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index ed750573f..b70a2e463 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -321,10 +321,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: 'Create Config File' env: - NIGHTLY_STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -354,10 +354,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) displayName: 'Create ADLS Config File' env: - NIGHTLY_STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(AZTEST_ADLS_KEY) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_TYPE: 'adls' + STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -387,9 +387,9 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_sas_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create SAS Config File" env: - NIGHTLY_STO_BLOB_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_SAS: $(NIGHTLY_STO_ACC_SAS) - ACCOUNT_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) + STO_ACC_SAS: $(NIGHTLY_STO_ACC_SAS) + STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -472,10 +472,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create MSI Config File" env: - NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) - NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) + STO_MSI_APP_ID: $(AZTEST_APP_ID) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -500,10 +500,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create MSI Config File" env: - NIGHTLY_STO_BLOB_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - NIGHTLY_MSI_APP_ID: $(AZTEST_APP_ID) - ACCOUNT_TYPE: 'adls' - ACCOUNT_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + STO_MSI_APP_ID: $(AZTEST_APP_ID) + STO_ACC_TYPE: 'adls' + STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false diff --git a/blobfuse2-perf.yaml b/blobfuse2-perf.yaml index 5c053d498..2794e4b57 100644 --- a/blobfuse2-perf.yaml +++ b/blobfuse2-perf.yaml @@ -124,10 +124,10 @@ stages: cat $(BLOBFUSE2_CFG) displayName: "Generate v2 Config File" env: - NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' continueOnError: false - script: | @@ -145,10 +145,10 @@ stages: echo "---------------------------------------------------" displayName: "Generate v2 Config File for File vs Block" env: - NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' continueOnError: false - script: | @@ -351,10 +351,10 @@ stages: cat $(BLOBFUSE2_CFG) displayName: "Generate v2 Config File" env: - NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' continueOnError: false - script: | @@ -383,10 +383,10 @@ stages: python3 $(WORK_DIR)/test/perf_test/resnet50_classify.py --dataset='$(MOUNT_DIR)/data' --job='main' --log=$(ROOT_DIR)/blobfuse2-perf.json displayName: "Blobfuse2 ResNet50 on Main" env: - NIGHTLY_STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) - NIGHTLY_STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) - ACCOUNT_TYPE: 'block' - ACCOUNT_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(PERF_WEEKLY_STO_BLOB_ACC_NAME) + STO_ACC_KEY: $(PERF_WEEKLY_STO_BLOB_ACC_KEY) + STO_ACC_TYPE: 'block' + STO_ACC_ENDPOINT: 'https://$(PERF_WEEKLY_STO_BLOB_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false continueOnError: false diff --git a/testdata/config/azure_block_perf.yaml b/testdata/config/azure_block_perf.yaml index 63b12571d..048ae6e65 100644 --- a/testdata/config/azure_block_perf.yaml +++ b/testdata/config/azure_block_perf.yaml @@ -28,11 +28,11 @@ attr_cache: timeout-sec: 7200 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } block-list-on-mount-sec: 10 diff --git a/testdata/config/azure_cli.yaml b/testdata/config/azure_cli.yaml index 8c6d2ed33..ed9e46737 100644 --- a/testdata/config/azure_cli.yaml +++ b/testdata/config/azure_cli.yaml @@ -26,10 +26,10 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_BLOB_ACC_NAME } + account-name: { STO_ACC_NAME } mode: azcli container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/azure_key.yaml b/testdata/config/azure_key.yaml index ace8a47e4..42b2972fe 100644 --- a/testdata/config/azure_key.yaml +++ b/testdata/config/azure_key.yaml @@ -26,11 +26,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: { USE_HTTP } - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_bc.yaml b/testdata/config/azure_key_bc.yaml index f83a11b2d..77fc9da3f 100644 --- a/testdata/config/azure_key_bc.yaml +++ b/testdata/config/azure_key_bc.yaml @@ -22,11 +22,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: { USE_HTTP } - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_directio.yaml b/testdata/config/azure_key_directio.yaml index 563e945ae..dded1ce87 100644 --- a/testdata/config/azure_key_directio.yaml +++ b/testdata/config/azure_key_directio.yaml @@ -23,11 +23,11 @@ file_cache: cleanup-on-start: true azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: { USE_HTTP } - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_emptyfile.yaml b/testdata/config/azure_key_emptyfile.yaml index eadab3cc7..7faecd9b1 100644 --- a/testdata/config/azure_key_emptyfile.yaml +++ b/testdata/config/azure_key_emptyfile.yaml @@ -27,11 +27,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_hmon.yaml b/testdata/config/azure_key_hmon.yaml index ef37194b3..668339dce 100644 --- a/testdata/config/azure_key_hmon.yaml +++ b/testdata/config/azure_key_hmon.yaml @@ -26,11 +26,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: { USE_HTTP } - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_huge.yaml b/testdata/config/azure_key_huge.yaml index 7d3ea18bf..b3a5ed821 100644 --- a/testdata/config/azure_key_huge.yaml +++ b/testdata/config/azure_key_huge.yaml @@ -28,11 +28,11 @@ attr_cache: no-cache-on-list: false azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_lru_purge.yaml b/testdata/config/azure_key_lru_purge.yaml index 236c20cad..37ad6c90e 100644 --- a/testdata/config/azure_key_lru_purge.yaml +++ b/testdata/config/azure_key_lru_purge.yaml @@ -25,11 +25,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_perf.yaml b/testdata/config/azure_key_perf.yaml index 1e9b05393..204a1d073 100644 --- a/testdata/config/azure_key_perf.yaml +++ b/testdata/config/azure_key_perf.yaml @@ -27,11 +27,11 @@ attr_cache: timeout-sec: 7200 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } block-list-on-mount-sec: 10 diff --git a/testdata/config/azure_key_proxy.yaml b/testdata/config/azure_key_proxy.yaml index ab8b2df90..aa0039986 100644 --- a/testdata/config/azure_key_proxy.yaml +++ b/testdata/config/azure_key_proxy.yaml @@ -26,11 +26,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_key_symlink.yaml b/testdata/config/azure_key_symlink.yaml index f46abdf4e..43977b4f9 100644 --- a/testdata/config/azure_key_symlink.yaml +++ b/testdata/config/azure_key_symlink.yaml @@ -27,11 +27,11 @@ attr_cache: no-symlinks: false azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: { USE_HTTP } - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot diff --git a/testdata/config/azure_msi.yaml b/testdata/config/azure_msi.yaml index c05293481..133b53dbc 100644 --- a/testdata/config/azure_msi.yaml +++ b/testdata/config/azure_msi.yaml @@ -26,11 +26,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_BLOB_ACC_NAME } - appid: { NIGHTLY_MSI_APP_ID } + account-name: { STO_ACC_NAME } + appid: { STO_MSI_APP_ID } mode: msi container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/azure_msi_sysid.yaml b/testdata/config/azure_msi_sysid.yaml index 8b8fe41fc..8a08a557a 100644 --- a/testdata/config/azure_msi_sysid.yaml +++ b/testdata/config/azure_msi_sysid.yaml @@ -26,8 +26,8 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - account-name: { NIGHTLY_STO_BLOB_ACC_NAME } + type: { STO_ACC_TYPE } + account-name: { STO_ACC_NAME } mode: msi container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/azure_sas.yaml b/testdata/config/azure_sas.yaml index 7eeb88443..d45965648 100644 --- a/testdata/config/azure_sas.yaml +++ b/testdata/config/azure_sas.yaml @@ -26,11 +26,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_BLOB_ACC_NAME } - sas: { NIGHTLY_STO_ACC_SAS } + account-name: { STO_ACC_NAME } + sas: { STO_ACC_SAS } mode: sas container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/azure_sas_proxy.yaml b/testdata/config/azure_sas_proxy.yaml index 5719fcaaa..5b8e65e14 100644 --- a/testdata/config/azure_sas_proxy.yaml +++ b/testdata/config/azure_sas_proxy.yaml @@ -27,10 +27,10 @@ attr_cache: azstorage: type: block - endpoint: { ACCOUNT_ENDPOINT } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_BLOB_ACC_NAME } - sas: { NIGHTLY_STO_ACC_SAS } + account-name: { STO_ACC_NAME } + sas: { STO_ACC_SAS } mode: sas container: { 0 } tier: hot diff --git a/testdata/config/azure_spn.yaml b/testdata/config/azure_spn.yaml index 9746aa288..73aaf6480 100644 --- a/testdata/config/azure_spn.yaml +++ b/testdata/config/azure_spn.yaml @@ -26,8 +26,8 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false account-name: { NIGHTLY_SPN_ACC_NAME } mode: spn diff --git a/testdata/config/azure_spn_proxy.yaml b/testdata/config/azure_spn_proxy.yaml index 47b73f7d9..d1b98d841 100644 --- a/testdata/config/azure_spn_proxy.yaml +++ b/testdata/config/azure_spn_proxy.yaml @@ -27,7 +27,7 @@ attr_cache: azstorage: type: block - endpoint: { ACCOUNT_ENDPOINT } + endpoint: { STO_ACC_ENDPOINT } use-http: false account-name: { NIGHTLY_SPN_ACC_NAME } mode: spn diff --git a/testdata/config/azure_stream.yaml b/testdata/config/azure_stream.yaml index 33fa9edf7..c528136b2 100644 --- a/testdata/config/azure_stream.yaml +++ b/testdata/config/azure_stream.yaml @@ -24,11 +24,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/azure_stream_direct.yaml b/testdata/config/azure_stream_direct.yaml index 913f67100..503787100 100644 --- a/testdata/config/azure_stream_direct.yaml +++ b/testdata/config/azure_stream_direct.yaml @@ -22,11 +22,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/azure_stream_filename.yaml b/testdata/config/azure_stream_filename.yaml index 8d67b508e..f8bf22551 100644 --- a/testdata/config/azure_stream_filename.yaml +++ b/testdata/config/azure_stream_filename.yaml @@ -25,11 +25,10 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/azure_stream_filename_direct.yaml b/testdata/config/azure_stream_filename_direct.yaml index 9784f4d1c..546edf6bb 100644 --- a/testdata/config/azure_stream_filename_direct.yaml +++ b/testdata/config/azure_stream_filename_direct.yaml @@ -23,11 +23,11 @@ attr_cache: timeout-sec: 3600 azstorage: - type: { ACCOUNT_TYPE } - endpoint: { ACCOUNT_ENDPOINT } + type: { STO_ACC_TYPE } + endpoint: { STO_ACC_ENDPOINT } use-http: false - account-name: { NIGHTLY_STO_ACC_NAME } - account-key: { NIGHTLY_STO_ACC_KEY } + account-name: { STO_ACC_NAME } + account-key: { STO_ACC_KEY } mode: key container: { 0 } tier: hot \ No newline at end of file diff --git a/testdata/config/config_key.yaml b/testdata/config/config_key.yaml index 00906b038..0acb0b265 100644 --- a/testdata/config/config_key.yaml +++ b/testdata/config/config_key.yaml @@ -3,11 +3,11 @@ logging: file-path: "blobfuse2-logs.txt" type: base -account-name: { NIGHTLY_STO_ACC_NAME } +account-name: { STO_ACC_NAME } container-name: { 0 } auth: type: key - account-account-key: { NIGHTLY_STO_ACC_KEY } + account-account-key: { STO_ACC_KEY } components: - libfuse - loopbackfs diff --git a/testdata/config/config_msi.yaml b/testdata/config/config_msi.yaml index 965457610..cb543fcb8 100644 --- a/testdata/config/config_msi.yaml +++ b/testdata/config/config_msi.yaml @@ -3,7 +3,7 @@ logging: file-path: "blobfuse2-logs.txt" type: base -account-name: { NIGHTLY_STO_BLOB_ACC_NAME } +account-name: { STO_ACC_NAME } container-name: { 0 } auth: type: msi diff --git a/testdata/config/config_sas.yaml b/testdata/config/config_sas.yaml index 2f1733118..2b2f760e7 100644 --- a/testdata/config/config_sas.yaml +++ b/testdata/config/config_sas.yaml @@ -3,11 +3,11 @@ logging: file-path: "blobfuse2-logs.txt" type: base -account-name: { NIGHTLY_STO_BLOB_ACC_NAME } +account-name: { STO_ACC_NAME } container-name: { 0 } auth: type: sas - sas-token: { NIGHTLY_STO_ACC_SAS } + sas-token: { STO_ACC_SAS } components: - libfuse - loopbackfs From 1a0df1ccb3672c16a5091ef2be357c58c0dcc363 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 19 Feb 2025 07:30:45 +0000 Subject: [PATCH 74/87] * Use only 2 storage accounts, 1 for block and other for adls --- azure-pipeline-templates/bfusemon.yml | 6 +- azure-pipeline-templates/build-release.yml | 12 ++-- azure-pipeline-templates/build.yml | 20 +++---- azure-pipeline-templates/cleanup.yml | 10 ++-- .../release-distro-tests.yml | 12 ++-- blobfuse2-code-coverage.yaml | 36 +++++------ blobfuse2-nightly.yaml | 60 +++++++++---------- 7 files changed, 79 insertions(+), 77 deletions(-) diff --git a/azure-pipeline-templates/bfusemon.yml b/azure-pipeline-templates/bfusemon.yml index 9764d15f4..dfcc4e269 100644 --- a/azure-pipeline-templates/bfusemon.yml +++ b/azure-pipeline-templates/bfusemon.yml @@ -5,9 +5,9 @@ steps: displayName: Create Key Config File env: STO_ACC_TYPE: 'block' - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_KEY: $(BF2_BLK_ACC_KEY) + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false HMON_OUTPUT: $(WORK_DIR) diff --git a/azure-pipeline-templates/build-release.yml b/azure-pipeline-templates/build-release.yml index 8982b9f46..f17852b09 100644 --- a/azure-pipeline-templates/build-release.yml +++ b/azure-pipeline-templates/build-release.yml @@ -67,16 +67,16 @@ steps: echo $cnfFile touch $cnfFile echo "{" > $cnfFile - echo "\"block-acct\"": "\"$(AZTEST_BLOCK_ACC_NAME)\"", >> $cnfFile - echo "\"adls-acct\"": "\"$(AZTEST_ADLS_ACC_NAME)\"", >> $cnfFile + echo "\"block-acct\"": "\"$(BF2_BLK_ACC_NAME)\"", >> $cnfFile + echo "\"adls-acct\"": "\"$(BF2_ADLS_ACC_NAME)\"", >> $cnfFile echo "\"block-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile echo "\"adls-cont\"": "\"${{ parameters.container }}\"", >> $cnfFile - echo "\"block-key\"": "\"$(AZTEST_BLOCK_KEY)\"", >> $cnfFile - echo "\"adls-key\"": "\"$(AZTEST_ADLS_KEY)\"", >> $cnfFile - echo "\"block-sas\"": "\"$(AZTEST_BLOCK_SAS)\"", >> $cnfFile + echo "\"block-key\"": "\"$(BF2_BLK_ACC_KEY)\"", >> $cnfFile + echo "\"adls-key\"": "\"$(BF2_ADLS_ACC_KEY)\"", >> $cnfFile + echo "\"block-sas\"": "\"$(BF2_BLK_ACC_SAS)\"", >> $cnfFile echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile - echo "\"adls-sas\"": "\"$(AZTEST_ADLS_SAS)\"", >> $cnfFile + echo "\"adls-sas\"": "\"$(BF2_ADLS_ACC_SAS)\"", >> $cnfFile echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 5f5ec7c76..d58865c1b 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -149,32 +149,32 @@ steps: parameters: create_container: true account_type: "block" - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) + account_name: $(BF2_BLK_ACC_NAME) + account_key: $(BF2_BLK_ACC_KEY) # Create a Container in the adls storage account - template: 'container.yml' parameters: create_container: true account_type: "adls" - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) + account_name: $(BF2_ADLS_ACC_NAME) + account_key: $(BF2_ADLS_ACC_KEY) - script: | cnfFile=$HOME/azuretest.json echo $cnfFile touch $cnfFile echo "{" > $cnfFile - echo "\"block-acct\"": "\"$(AZTEST_BLOCK_ACC_NAME)\"", >> $cnfFile - echo "\"adls-acct\"": "\"$(AZTEST_ADLS_ACC_NAME)\"", >> $cnfFile + echo "\"block-acct\"": "\"$(BF2_BLK_ACC_NAME)\"", >> $cnfFile + echo "\"adls-acct\"": "\"$(BF2_ADLS_ACC_NAME)\"", >> $cnfFile echo "\"block-cont\"": "\"$(containerName)\"", >> $cnfFile echo "\"adls-cont\"": "\"$(containerName)\"", >> $cnfFile - echo "\"block-key\"": "\"$(AZTEST_BLOCK_KEY)\"", >> $cnfFile - echo "\"adls-key\"": "\"$(AZTEST_ADLS_KEY)\"", >> $cnfFile - echo "\"block-sas\"": "\"$(AZTEST_BLOCK_SAS)\"", >> $cnfFile + echo "\"block-key\"": "\"$(BF2_BLK_ACC_KEY)\"", >> $cnfFile + echo "\"adls-key\"": "\"$(BF2_ADLS_ACC_KEY)\"", >> $cnfFile + echo "\"block-sas\"": "\"$(BF2_BLK_ACC_SAS)\"", >> $cnfFile echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile - echo "\"adls-sas\"": "\"$(AZTEST_ADLS_SAS)\"", >> $cnfFile + echo "\"adls-sas\"": "\"$(BF2_ADLS_ACC_SAS)\"", >> $cnfFile echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile diff --git a/azure-pipeline-templates/cleanup.yml b/azure-pipeline-templates/cleanup.yml index 292fc0c53..fdd6ec672 100755 --- a/azure-pipeline-templates/cleanup.yml +++ b/azure-pipeline-templates/cleanup.yml @@ -16,6 +16,8 @@ steps: - script: | sudo umount -f $(MOUNT_DIR) + sleep 10 + ps aux | grep blobfuse2 sudo kill -9 `pidof blobfuse2` &> /dev/null rm -rf $(MOUNT_DIR)/* rm -rf $(TEMP_DIR)/* @@ -30,8 +32,8 @@ steps: parameters: delete_container: true account_type: "block" - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) + account_name: $(BF2_BLK_ACC_NAME) + account_key: $(BF2_BLK_ACC_KEY) container_name: $(containerName) # delete the container in adls account @@ -40,6 +42,6 @@ steps: parameters: delete_container: true account_type: "adls" - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) + account_name: $(BF2_ADLS_ACC_NAME) + account_key: $(BF2_ADLS_ACC_KEY) container_name: $(containerName) \ No newline at end of file diff --git a/azure-pipeline-templates/release-distro-tests.yml b/azure-pipeline-templates/release-distro-tests.yml index c05588472..6f5c17cec 100644 --- a/azure-pipeline-templates/release-distro-tests.yml +++ b/azure-pipeline-templates/release-distro-tests.yml @@ -33,10 +33,10 @@ steps: blobfuse2 gen-test-config --config-file=${{ parameters.root_dir }}/azure-storage-fuse/testdata/config/azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.root_dir }}/block_blob_config.yaml displayName: 'Create Blob Config File' env: - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_KEY: $(BF2_BLK_ACC_KEY) STO_ACC_TYPE: 'block' - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false continueOnError: false @@ -92,10 +92,10 @@ steps: blobfuse2 gen-test-config --config-file=${{ parameters.root_dir }}/azure-storage-fuse/testdata/config/azure_key.yaml --container-name=${{ parameters.container }} --temp-path=${{ parameters.temp_dir }} --output-file=${{ parameters.root_dir }}/adls_config.yaml displayName: 'Create ADLS Config File' env: - STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_NAME: $(BF2_ADLS_ACC_NAME) + STO_ACC_KEY: $(BF2_ADLS_ACC_KEY) STO_ACC_TYPE: 'adls' - STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: false USE_HTTP: false continueOnError: false diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index cc69048ad..83fc23f48 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -71,10 +71,10 @@ stages: cd $(WORK_DIR) ./blobfuse2.test -test.v -test.coverprofile=$(WORK_DIR)/blobfuse2_gentest1.cov gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) env: - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_KEY: $(BF2_BLK_ACC_KEY) STO_ACC_TYPE: 'block' - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - Block Blob' @@ -128,10 +128,10 @@ stages: cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) env: - STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_NAME: $(BF2_ADLS_ACC_NAME) + STO_ACC_KEY: $(BF2_ADLS_ACC_KEY) STO_ACC_TYPE: 'adls' - STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - ADLS' @@ -186,10 +186,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: 'Create Config File - Block Blob' env: - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_KEY: $(BF2_BLK_ACC_KEY) STO_ACC_TYPE: 'block' - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false @@ -408,10 +408,10 @@ stages: workingDirectory: $(WORK_DIR) displayName: "CLI : Secure Config" env: - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_KEY: $(BF2_BLK_ACC_KEY) STO_ACC_TYPE: 'block' - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false @@ -478,10 +478,10 @@ stages: cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) env: - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_KEY: $(BF2_BLK_ACC_KEY) STO_ACC_TYPE: 'block' - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - Block Blob Proxy' @@ -509,10 +509,10 @@ stages: cd $(WORK_DIR) $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) env: - STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_NAME: $(BF2_ADLS_ACC_NAME) + STO_ACC_KEY: $(BF2_ADLS_ACC_KEY) STO_ACC_TYPE: 'adls' - STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: false USE_HTTP: false displayName: 'Create Config File - ADLS Proxy' diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index b70a2e463..f99605048 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -130,11 +130,11 @@ stages: parameters: service: 'BlockBlob' account_type: 'block' - account_endpoint: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + account_endpoint: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) - account_sas: $(NIGHTLY_STO_ACC_SAS) + account_name: $(BF2_BLK_ACC_NAME) + account_key: $(BF2_BLK_ACC_KEY) + account_sas: $(BF2_BLK_ACC_SAS) client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) @@ -234,8 +234,8 @@ stages: parameters: generate_adls_sas: true account_type: "adls" - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) + account_name: $(BF2_ADLS_ACC_NAME) + account_key: $(BF2_ADLS_ACC_KEY) # ------------------------------------------------------- - ${{ if eq(parameters.exhaustive_test, true) }}: @@ -243,10 +243,10 @@ stages: parameters: service: 'ADLS' account_type: 'adls' - account_endpoint: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + account_endpoint: 'https://$(BF2_ADLS_ACC_NAME).dfs.core.windows.net' adls: true - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) + account_name: $(BF2_ADLS_ACC_NAME) + account_key: $(BF2_ADLS_ACC_KEY) account_sas: $(GENERATED_ADLS_CONTAINER_SAS) client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) @@ -321,10 +321,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: 'Create Config File' env: - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_KEY: $(NIGHTLY_STO_BLOB_ACC_KEY) + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_KEY: $(BF2_BLK_ACC_KEY) STO_ACC_TYPE: 'block' - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -354,10 +354,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_key_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_ADLS_CFG) displayName: 'Create ADLS Config File' env: - STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) - STO_ACC_KEY: $(AZTEST_ADLS_KEY) + STO_ACC_NAME: $(BF2_ADLS_ACC_NAME) + STO_ACC_KEY: $(BF2_ADLS_ACC_KEY) STO_ACC_TYPE: 'adls' - STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -387,9 +387,9 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_sas_proxy.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create SAS Config File" env: - STO_ACC_NAME: $(NIGHTLY_STO_BLOB_ACC_NAME) - STO_ACC_SAS: $(NIGHTLY_STO_ACC_SAS) - STO_ACC_ENDPOINT: 'https://$(NIGHTLY_STO_BLOB_ACC_NAME).blob.core.windows.net' + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) + STO_ACC_SAS: $(BF2_BLK_ACC_SAS) + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -472,10 +472,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create MSI Config File" env: - STO_ACC_NAME: $(AZTEST_BLOCK_ACC_NAME) + STO_ACC_NAME: $(BF2_BLK_ACC_NAME) STO_MSI_APP_ID: $(AZTEST_APP_ID) STO_ACC_TYPE: 'block' - STO_ACC_ENDPOINT: 'https://$(AZTEST_BLOCK_ACC_NAME).blob.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -500,10 +500,10 @@ stages: $(WORK_DIR)/blobfuse2 gen-test-config --config-file=azure_msi.yaml --container-name=$(containerName) --temp-path=$(TEMP_DIR) --output-file=$(BLOBFUSE2_CFG) displayName: "Create MSI Config File" env: - STO_ACC_NAME: $(AZTEST_ADLS_ACC_NAME) + STO_ACC_NAME: $(BF2_ADLS_ACC_NAME) STO_MSI_APP_ID: $(AZTEST_APP_ID) STO_ACC_TYPE: 'adls' - STO_ACC_ENDPOINT: 'https://$(AZTEST_ADLS_ACC_NAME).dfs.core.windows.net' + STO_ACC_ENDPOINT: 'https://$(BF2_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -640,8 +640,8 @@ stages: config_file: $(BLOBFUSE2_CFG) idstring: block_blob_key adls: false - account_name: $(NIGHTLY_STO_BLOB_ACC_NAME) - account_key: $(NIGHTLY_STO_BLOB_ACC_KEY) + account_name: $(BF2_BLK_ACC_NAME) + account_key: $(BF2_BLK_ACC_KEY) account_type: block distro_name: $(distro_version) quick_test: true @@ -655,8 +655,8 @@ stages: config_file: $(BLOBFUSE2_CFG) idstring: adls_key adls: true - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) + account_name: $(BF2_ADLS_ACC_NAME) + account_key: $(BF2_ADLS_ACC_KEY) account_type: adls distro_name: $(distro_version) quick_test: true @@ -725,8 +725,8 @@ stages: parameters: config_file: $(BLOBFUSE2_CFG) idstring: BLOCK - account_name: $(AZTEST_BLOCK_ACC_NAME) - account_key: $(AZTEST_BLOCK_KEY) + account_name: $(BF2_BLK_ACC_NAME) + account_key: $(BF2_BLK_ACC_KEY) account_type: block verbose_log: ${{ parameters.verbose_log }} @@ -735,8 +735,8 @@ stages: parameters: config_file: $(BLOBFUSE2_CFG) idstring: ADLS - account_name: $(AZTEST_ADLS_ACC_NAME) - account_key: $(AZTEST_ADLS_KEY) + account_name: $(BF2_ADLS_ACC_NAME) + account_key: $(BF2_ADLS_ACC_KEY) account_type: adls verbose_log: ${{ parameters.verbose_log }} From c2366c56b88f76da51590b84fbe6ce39ed21f113 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 19 Feb 2025 07:35:15 +0000 Subject: [PATCH 75/87] Fix typo --- azure-pipeline-templates/data-consistency.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-consistency.yml index f7f347058..4bae7d8d8 100644 --- a/azure-pipeline-templates/data-consistency.yml +++ b/azure-pipeline-templates/data-consistency.yml @@ -30,7 +30,7 @@ steps: env: STO_ACC_NAME: ${{ parameters.account_name }} STO_ACC_KEY: ${{ parameters.account_key }} - STO_ACC_TYPE: ${{ parameters.STO_ACC_TYPE }} + STO_ACC_TYPE: ${{ parameters.account_type }} VERBOSE_LOG: ${{ parameters.verbose_log }} continueOnError: false @@ -142,7 +142,7 @@ steps: env: STO_ACC_NAME: ${{ parameters.account_name }} STO_ACC_KEY: ${{ parameters.account_key }} - STO_ACC_TYPE: ${{ parameters.STO_ACC_TYPE }} + STO_ACC_TYPE: ${{ parameters.account_type }} VERBOSE_LOG: ${{ parameters.verbose_log }} # Mount using stream config. From 0ec6bae7079b96e87894dc36baead0a798a12ec3 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 19 Feb 2025 10:58:58 +0000 Subject: [PATCH 76/87] Check trigger is working or not --- blobfuse2-ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index bd24d7edb..38e6a71b4 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -11,6 +11,7 @@ pr: trigger: - main - ci/* +- syeleti/* jobs: # Ubuntu based test suite From 5a1746e3a6bfddbd37e61a9d518aea6429e964b5 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 09:16:35 +0000 Subject: [PATCH 77/87] Move containers for listing huge number of files --- blobfuse2-nightly.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index f99605048..42df403e4 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -140,7 +140,7 @@ stages: client_secret: $(AZTEST_SECRET) config: $(BLOBFUSE2_CFG) stress_dir: $(BLOBFUSE2_STRESS_DIR) - huge_container: 'testcnt1' + huge_container: 'million-files' quick_stress: ${{ parameters.quick_stress }} test_key_credential: true test_sas_credential: true @@ -253,7 +253,7 @@ stages: client_secret: $(AZTEST_SECRET) config: $(BLOBFUSE2_ADLS_CFG) stress_dir: $(BLOBFUSE2_STRESS_DIR) - huge_container: 'testcnt' + huge_container: 'million-files' quick_stress: ${{ parameters.quick_stress }} test_key_credential: true test_sas_credential: true From 578791184e279c431cdce1e7cee9c8b85f60e066 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 10:02:35 +0000 Subject: [PATCH 78/87] Move generation of ADLS SAS to the setup of pipeline --- azure-pipeline-templates/build.yml | 12 +++++++++++- blobfuse2-nightly.yaml | 10 ---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index d58865c1b..e224c2e06 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -160,6 +160,16 @@ steps: account_name: $(BF2_ADLS_ACC_NAME) account_key: $(BF2_ADLS_ACC_KEY) + #----------------------------------------------------------- + # Generate SAS for created container in ADLS with all the permissions. + # Reason: the account SAS is not sufficient to change the ACL's + - template: 'azure-pipeline-templates/container.yml' + parameters: + generate_adls_sas: true + account_type: "adls" + account_name: $(BF2_ADLS_ACC_NAME) + account_key: $(BF2_ADLS_ACC_KEY) + - script: | cnfFile=$HOME/azuretest.json echo $cnfFile @@ -174,7 +184,7 @@ steps: echo "\"block-sas\"": "\"$(BF2_BLK_ACC_SAS)\"", >> $cnfFile echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile - echo "\"adls-sas\"": "\"$(BF2_ADLS_ACC_SAS)\"", >> $cnfFile + echo "\"adls-sas\"": "\"$(GENERATED_ADLS_CONTAINER_SAS)\"", >> $cnfFile echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 42df403e4..a91318bce 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -226,16 +226,6 @@ stages: parameters: skip_ut: true # Skip UT because Block Blob set runs it skip_azcli: "false" - - #----------------------------------------------------------- - # Generate SAS for created container in ADLS with all the permissions. - # Reason: the account SAS is not sufficient to change the ACL's - - template: 'azure-pipeline-templates/container.yml' - parameters: - generate_adls_sas: true - account_type: "adls" - account_name: $(BF2_ADLS_ACC_NAME) - account_key: $(BF2_ADLS_ACC_KEY) # ------------------------------------------------------- - ${{ if eq(parameters.exhaustive_test, true) }}: From 83eddebd1481a44f9e28d05a882c89c9f464300a Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 10:05:39 +0000 Subject: [PATCH 79/87] Move generation of ADLS SAS to the setup of pipeline --- azure-pipeline-templates/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index e224c2e06..d47a6d70e 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -163,7 +163,7 @@ steps: #----------------------------------------------------------- # Generate SAS for created container in ADLS with all the permissions. # Reason: the account SAS is not sufficient to change the ACL's - - template: 'azure-pipeline-templates/container.yml' + - template: 'container.yml' parameters: generate_adls_sas: true account_type: "adls" From 13a5523a4d708079adc668f420524227971dfc55 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 12:46:13 +0000 Subject: [PATCH 80/87] Disable tests for checking auth using http as new storage accounts are not allowing to enable http on acnt may be due to security issues --- azure-pipeline-templates/build.yml | 2 +- blobfuse2-nightly.yaml | 4 +- component/azstorage/azauth_test.go | 121 +++++++++++++++-------------- 3 files changed, 64 insertions(+), 63 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index d47a6d70e..f1cd23af6 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -200,7 +200,7 @@ steps: # azcli login, This is only for ubuntu distributions, where we create containers for the job - script: | - az login --identity --username $(ACLI_BLOBFUSE_MSI_APP_ID) + az login --identity --username $(BF2_MSI_CLIENT_ID) displayName: 'Azure CLI login' condition: eq(variables['distro'], 'ubuntu') diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index a91318bce..e8e3e8582 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -463,7 +463,7 @@ stages: displayName: "Create MSI Config File" env: STO_ACC_NAME: $(BF2_BLK_ACC_NAME) - STO_MSI_APP_ID: $(AZTEST_APP_ID) + STO_MSI_APP_ID: $(BF2_MSI_CLIENT_ID) STO_ACC_TYPE: 'block' STO_ACC_ENDPOINT: 'https://$(BF2_BLK_ACC_NAME).blob.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} @@ -491,7 +491,7 @@ stages: displayName: "Create MSI Config File" env: STO_ACC_NAME: $(BF2_ADLS_ACC_NAME) - STO_MSI_APP_ID: $(AZTEST_APP_ID) + STO_MSI_APP_ID: $(BF2_MSI_CLIENT_ID) STO_ACC_TYPE: 'adls' STO_ACC_ENDPOINT: 'https://$(BF2_ADLS_ACC_NAME).dfs.core.windows.net' VERBOSE_LOG: ${{ parameters.verbose_log }} diff --git a/component/azstorage/azauth_test.go b/component/azstorage/azauth_test.go index 30d05590f..226fb33d1 100644 --- a/component/azstorage/azauth_test.go +++ b/component/azstorage/azauth_test.go @@ -271,21 +271,22 @@ func (suite *authTestSuite) TestBlockSharedKey() { } suite.validateStorageTest("TestBlockSharedKey", stgConfig) } -func (suite *authTestSuite) TestHttpBlockSharedKey() { - defer suite.cleanupTest() - stgConfig := AzStorageConfig{ - container: storageTestConfigurationParameters.BlockContainer, - authConfig: azAuthConfig{ - AuthMode: EAuthType.KEY(), - AccountType: EAccountType.BLOCK(), - AccountName: storageTestConfigurationParameters.BlockAccount, - AccountKey: storageTestConfigurationParameters.BlockKey, - UseHTTP: true, - Endpoint: generateEndpoint(true, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()), - }, - } - suite.validateStorageTest("TestHttpBlockSharedKey", stgConfig) -} + +// func (suite *authTestSuite) TestHttpBlockSharedKey() { +// defer suite.cleanupTest() +// stgConfig := AzStorageConfig{ +// container: storageTestConfigurationParameters.BlockContainer, +// authConfig: azAuthConfig{ +// AuthMode: EAuthType.KEY(), +// AccountType: EAccountType.BLOCK(), +// AccountName: storageTestConfigurationParameters.BlockAccount, +// AccountKey: storageTestConfigurationParameters.BlockKey, +// UseHTTP: true, +// Endpoint: generateEndpoint(true, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()), +// }, +// } +// suite.validateStorageTest("TestHttpBlockSharedKey", stgConfig) +// } func (suite *authTestSuite) TestAdlsInvalidSharedKey() { defer suite.cleanupTest() @@ -324,21 +325,21 @@ func (suite *authTestSuite) TestAdlsSharedKey() { suite.validateStorageTest("TestAdlsSharedKey", stgConfig) } -func (suite *authTestSuite) TestHttpAdlsSharedKey() { - defer suite.cleanupTest() - stgConfig := AzStorageConfig{ - container: storageTestConfigurationParameters.AdlsContainer, - authConfig: azAuthConfig{ - AuthMode: EAuthType.KEY(), - AccountType: EAccountType.ADLS(), - AccountName: storageTestConfigurationParameters.AdlsAccount, - AccountKey: storageTestConfigurationParameters.AdlsKey, - UseHTTP: true, - Endpoint: generateEndpoint(true, storageTestConfigurationParameters.AdlsAccount, EAccountType.ADLS()), - }, - } - suite.validateStorageTest("TestHttpAdlsSharedKey", stgConfig) -} +// func (suite *authTestSuite) TestHttpAdlsSharedKey() { +// defer suite.cleanupTest() +// stgConfig := AzStorageConfig{ +// container: storageTestConfigurationParameters.AdlsContainer, +// authConfig: azAuthConfig{ +// AuthMode: EAuthType.KEY(), +// AccountType: EAccountType.ADLS(), +// AccountName: storageTestConfigurationParameters.AdlsAccount, +// AccountKey: storageTestConfigurationParameters.AdlsKey, +// UseHTTP: true, +// Endpoint: generateEndpoint(true, storageTestConfigurationParameters.AdlsAccount, EAccountType.ADLS()), +// }, +// } +// suite.validateStorageTest("TestHttpAdlsSharedKey", stgConfig) +// } func (suite *authTestSuite) TestBlockInvalidSasKey() { defer suite.cleanupTest() @@ -377,21 +378,21 @@ func (suite *authTestSuite) TestBlockSasKey() { suite.validateStorageTest("TestBlockSasKey", stgConfig) } -func (suite *authTestSuite) TestHttpBlockSasKey() { - defer suite.cleanupTest() - stgConfig := AzStorageConfig{ - container: storageTestConfigurationParameters.BlockContainer, - authConfig: azAuthConfig{ - AuthMode: EAuthType.SAS(), - AccountType: EAccountType.BLOCK(), - AccountName: storageTestConfigurationParameters.BlockAccount, - SASKey: storageTestConfigurationParameters.BlockSas, - UseHTTP: true, - Endpoint: generateEndpoint(true, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()), - }, - } - suite.validateStorageTest("TestHttpBlockSasKey", stgConfig) -} +// func (suite *authTestSuite) TestHttpBlockSasKey() { +// defer suite.cleanupTest() +// stgConfig := AzStorageConfig{ +// container: storageTestConfigurationParameters.BlockContainer, +// authConfig: azAuthConfig{ +// AuthMode: EAuthType.SAS(), +// AccountType: EAccountType.BLOCK(), +// AccountName: storageTestConfigurationParameters.BlockAccount, +// SASKey: storageTestConfigurationParameters.BlockSas, +// UseHTTP: true, +// Endpoint: generateEndpoint(true, storageTestConfigurationParameters.BlockAccount, EAccountType.BLOCK()), +// }, +// } +// suite.validateStorageTest("TestHttpBlockSasKey", stgConfig) +// } func (suite *authTestSuite) TestBlockContSasKey() { defer suite.cleanupTest() @@ -507,21 +508,21 @@ func (suite *authTestSuite) TestAdlsSasKey() { suite.validateStorageTest("TestAdlsSasKey", stgConfig) } -func (suite *authTestSuite) TestHttpAdlsSasKey() { - defer suite.cleanupTest() - stgConfig := AzStorageConfig{ - container: storageTestConfigurationParameters.AdlsContainer, - authConfig: azAuthConfig{ - AuthMode: EAuthType.SAS(), - AccountType: EAccountType.ADLS(), - AccountName: storageTestConfigurationParameters.AdlsAccount, - SASKey: storageTestConfigurationParameters.AdlsSas, - UseHTTP: true, - Endpoint: generateEndpoint(true, storageTestConfigurationParameters.AdlsAccount, EAccountType.ADLS()), - }, - } - suite.validateStorageTest("TestHttpAdlsSasKey", stgConfig) -} +// func (suite *authTestSuite) TestHttpAdlsSasKey() { +// defer suite.cleanupTest() +// stgConfig := AzStorageConfig{ +// container: storageTestConfigurationParameters.AdlsContainer, +// authConfig: azAuthConfig{ +// AuthMode: EAuthType.SAS(), +// AccountType: EAccountType.ADLS(), +// AccountName: storageTestConfigurationParameters.AdlsAccount, +// SASKey: storageTestConfigurationParameters.AdlsSas, +// UseHTTP: true, +// Endpoint: generateEndpoint(true, storageTestConfigurationParameters.AdlsAccount, EAccountType.ADLS()), +// }, +// } +// suite.validateStorageTest("TestHttpAdlsSasKey", stgConfig) +// } // func (suite *authTestSuite) TestAdlsDirSasKey() { // defer suite.cleanupTest() From 57e1592e6e72dbb30ed2727310de1e5c3939a98b Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 14:24:01 +0000 Subject: [PATCH 81/87] make ADLS SAS getting created only when ubuntu distros --- azure-pipeline-templates/build.yml | 2 +- azure-pipeline-templates/container.yml | 2 +- blobfuse2-nightly.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index f1cd23af6..9a6bea1cd 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -184,7 +184,7 @@ steps: echo "\"block-sas\"": "\"$(BF2_BLK_ACC_SAS)\"", >> $cnfFile echo "\"block-cont-sas-ubn-18\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_18)\"", >> $cnfFile echo "\"block-cont-sas-ubn-20\"": "\"$(AZTEST_BLOCK_CONT_SAS_UBN_20)\"", >> $cnfFile - echo "\"adls-sas\"": "\"$(GENERATED_ADLS_CONTAINER_SAS)\"", >> $cnfFile + echo "\"adls-sas\"": "\"$(BF2_ADLS_ACC_SAS)\"", >> $cnfFile echo "\"msi-appid\"": "\"$(AZTEST_APP_ID)\"", >> $cnfFile echo "\"msi-resid\"": "\"$(AZTEST_RES_ID)\"", >> $cnfFile echo "\"msi-objid\"": "\"$(AZTEST_OBJ_ID)\"", >> $cnfFile diff --git a/azure-pipeline-templates/container.yml b/azure-pipeline-templates/container.yml index d2ba70a3c..6780f8d94 100644 --- a/azure-pipeline-templates/container.yml +++ b/azure-pipeline-templates/container.yml @@ -58,7 +58,7 @@ steps: --permissions acdelmoprw \ --expiry $end \ -o tsv` - echo "##vso[task.setvariable variable=GENERATED_ADLS_CONTAINER_SAS]$sas" + echo "##vso[task.setvariable variable=BF2_ADLS_ACC_SAS]$sas" displayName: 'Creating ADLS SAS for ${{ parameters.account_type }} Container' condition: eq(variables['distro'], 'ubuntu') diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index e8e3e8582..32f3448d5 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -237,7 +237,7 @@ stages: adls: true account_name: $(BF2_ADLS_ACC_NAME) account_key: $(BF2_ADLS_ACC_KEY) - account_sas: $(GENERATED_ADLS_CONTAINER_SAS) + account_sas: $(BF2_ADLS_ACC_SAS) client_id: $(AZTEST_CLIENT) tenant_id: $(AZTEST_TENANT) client_secret: $(AZTEST_SECRET) From 5e8a3f0e2b327387e3a175ceb65eddd0fee1170f Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 15:11:10 +0000 Subject: [PATCH 82/87] update code coverage --- blobfuse2-code-coverage.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index 83fc23f48..d4e13965d 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -15,10 +15,14 @@ stages: matrix: Ubuntu-20: AgentName: 'blobfuse-ubuntu20' + distro: 'ubuntu' + distro_version: 'ubuntu_20' fuselib: 'libfuse-dev' tags: 'fuse2' Ubuntu-22: AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' fuselib: 'libfuse3-dev' tags: 'fuse3' From b70ac010a959253a417319046cc753e70b65f096 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 15:51:39 +0000 Subject: [PATCH 83/87] Refactor --- azure-pipeline-templates/build.yml | 2 +- blobfuse2-ci.yaml | 7 ++-- blobfuse2-nightly.yaml | 61 ++++++++++++++++++++++++++++-- 3 files changed, 63 insertions(+), 7 deletions(-) diff --git a/azure-pipeline-templates/build.yml b/azure-pipeline-templates/build.yml index 9a6bea1cd..41ad271a8 100755 --- a/azure-pipeline-templates/build.yml +++ b/azure-pipeline-templates/build.yml @@ -202,7 +202,7 @@ steps: - script: | az login --identity --username $(BF2_MSI_CLIENT_ID) displayName: 'Azure CLI login' - condition: eq(variables['distro'], 'ubuntu') + condition: eq('${{ parameters.skip_azcli }}', 'false') # Running unit tests - task: Go@0 diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 38e6a71b4..1e0aca770 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -11,6 +11,10 @@ pr: trigger: - main - ci/* +- vibhansa/* +- akanksha/* +- sourav/* +- ashruti/* - syeleti/* jobs: @@ -140,9 +144,6 @@ jobs: failOnStderr: true workingDirectory: $(WORK_DIR) - # checks for bfusemon - - template: 'azure-pipeline-templates/bfusemon.yml' - # CleanUp the mount and delete the containers. - template: 'azure-pipeline-templates/cleanup.yml' parameters: diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 32f3448d5..cc010dbd2 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -46,7 +46,11 @@ parameters: displayName: 'Verbose Log' type: boolean default: false - + + - name: fusemon + displayName: 'Run tests on bfusemon' + type: boolean + default: false stages: - ${{ if eq(parameters.base_test, true) }}: @@ -660,7 +664,7 @@ stages: delete_containers: true - - stage: DataConsistency + - stage: DataIntegrity jobs: # Ubuntu Tests - job: Set_1 @@ -733,4 +737,55 @@ stages: - template: azure-pipeline-templates/cleanup.yml parameters: unmount: true - delete_containers: true \ No newline at end of file + delete_containers: true + +# Run bfusemon tests optionally when user invokes. + - ${{ if eq(parameters.fusemon, true) }}: + - stage: fusemon + jobs: + # Ubuntu Tests + - job: Set_1 + timeoutInMinutes: 300 + strategy: + matrix: + Ubuntu-22: + AgentName: 'blobfuse-ubuntu22' + distro: 'ubuntu' + distro_version: 'ubuntu_22_x86' + fuselib: 'libfuse3-dev' + tags: 'fuse3' + + pool: + name: "blobfuse-ubuntu-pool" + demands: + - ImageOverride -equals $(AgentName) + + variables: + - group: NightlyBlobFuse + - name: MOUNT_DIR + value: '$(Pipeline.Workspace)/blob_mnt' + - name: TEMP_DIR + value: '$(Pipeline.Workspace)/blobfuse2_tmp' + - name: BLOBFUSE2_CFG + value: '$(Pipeline.Workspace)/blobfuse2.yaml' + - name: BLOBFUSE2_ADLS_CFG + value: '$(Pipeline.Workspace)/blobfuse2.adls.yaml' + - name: skipComponentGovernanceDetection + value: true + - name: GOPATH + value: '$(Pipeline.Workspace)/go' + - name: ROOT_DIR + value: '$(System.DefaultWorkingDirectory)' + - name: WORK_DIR + value: '$(System.DefaultWorkingDirectory)/azure-storage-fuse' + + steps: + # ------------------------------------------------------- + # Pull and build the code and create the containers + - template: 'azure-pipeline-templates/build.yml' + parameters: + skip_ut: true + + + # checks for bfusemon + - template: 'azure-pipeline-templates/bfusemon.yml' \ No newline at end of file From 39d8c5dbe6d59f17acef22f19cdfcb4cce06e799 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Tue, 4 Mar 2025 15:57:56 +0000 Subject: [PATCH 84/87] refactor --- .../{data-consistency.yml => data-integrity.yml} | 0 azure-pipeline-templates/data.yml | 5 +++++ blobfuse2-nightly.yaml | 10 +++++++--- 3 files changed, 12 insertions(+), 3 deletions(-) rename azure-pipeline-templates/{data-consistency.yml => data-integrity.yml} (100%) diff --git a/azure-pipeline-templates/data-consistency.yml b/azure-pipeline-templates/data-integrity.yml similarity index 100% rename from azure-pipeline-templates/data-consistency.yml rename to azure-pipeline-templates/data-integrity.yml diff --git a/azure-pipeline-templates/data.yml b/azure-pipeline-templates/data.yml index 56584362c..f8cb50e19 100644 --- a/azure-pipeline-templates/data.yml +++ b/azure-pipeline-templates/data.yml @@ -45,6 +45,11 @@ steps: echo "*************************Generating medium files********************************" generate_random_file "1M" displayName: "Generate files with random data using dd command" + + - bash: | + ls -lrt + displayName: "List the generated Files" + workingDirectory: $(DATA_DIR) - bash: | md5sum * | tee checklist.chk diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index cc010dbd2..c4fa8372f 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -48,7 +48,7 @@ parameters: default: false - name: fusemon - displayName: 'Run tests on bfusemon' + displayName: 'Run tests for bfusemon' type: boolean default: false @@ -786,6 +786,10 @@ stages: parameters: skip_ut: true - # checks for bfusemon - - template: 'azure-pipeline-templates/bfusemon.yml' \ No newline at end of file + - template: 'azure-pipeline-templates/bfusemon.yml' + + - template: azure-pipeline-templates/cleanup.yml + parameters: + unmount: true + delete_containers: true \ No newline at end of file From 487d494c2daf844968f8be172da90524b43f0003 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 5 Mar 2025 03:56:37 +0000 Subject: [PATCH 85/87] Finalise changes --- blobfuse2-ci.yaml | 1 - blobfuse2-nightly.yaml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/blobfuse2-ci.yaml b/blobfuse2-ci.yaml index 1e0aca770..6c7021ceb 100644 --- a/blobfuse2-ci.yaml +++ b/blobfuse2-ci.yaml @@ -86,7 +86,6 @@ jobs: parameters: skip_azcli: "false" skip_ut: "false" - skip_bfusemon: false # Code lint checks (Static-analysis) - script: | diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index c4fa8372f..0da8ff0cd 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -785,6 +785,7 @@ stages: - template: 'azure-pipeline-templates/build.yml' parameters: skip_ut: true + skip_bfusemon: false # checks for bfusemon - template: 'azure-pipeline-templates/bfusemon.yml' From 4870ccb1d4c39bc5edf5a2a8df0c4a827f8447e3 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 5 Mar 2025 04:05:33 +0000 Subject: [PATCH 86/87] typo --- blobfuse2-nightly.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/blobfuse2-nightly.yaml b/blobfuse2-nightly.yaml index 0da8ff0cd..899257567 100755 --- a/blobfuse2-nightly.yaml +++ b/blobfuse2-nightly.yaml @@ -715,7 +715,7 @@ stages: parameters: skip_ut: true # Check data consistency for block accounts - - template: 'azure-pipeline-templates/data-consistency.yml' + - template: 'azure-pipeline-templates/data-integrity.yml' parameters: config_file: $(BLOBFUSE2_CFG) idstring: BLOCK @@ -725,7 +725,7 @@ stages: verbose_log: ${{ parameters.verbose_log }} # Check data consistency for adls accounts - - template: 'azure-pipeline-templates/data-consistency.yml' + - template: 'azure-pipeline-templates/data-integrity.yml' parameters: config_file: $(BLOBFUSE2_CFG) idstring: ADLS From efa37580a5ad36b720680ecfb88afc066a774140 Mon Sep 17 00:00:00 2001 From: Srinivas Yeleti Date: Wed, 5 Mar 2025 04:07:42 +0000 Subject: [PATCH 87/87] refactor --- blobfuse2-code-coverage.yaml | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/blobfuse2-code-coverage.yaml b/blobfuse2-code-coverage.yaml index b271e209e..30cfd641f 100644 --- a/blobfuse2-code-coverage.yaml +++ b/blobfuse2-code-coverage.yaml @@ -538,20 +538,20 @@ stages: workingDirectory: $(WORK_DIR) displayName: "ADLS Coverage Proxy" - # ------------------------------------------------------- - # Coverage report consolidation - - script: | - echo 'mode: count' > ./blobfuse2_coverage_raw.rpt - tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt - cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" | grep -v "exported/exported.go" | grep -v "component/block_cache/stream.go" | grep -v "component/azstorage/azAuthWorkloadIdentity.go" > ./blobfuse2_coverage.rpt - go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt - go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html - go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html - cp ./*.rpt $(Build.ArtifactStagingDirectory)/ - cp ./*.cov $(Build.ArtifactStagingDirectory)/ - cp ./*.html $(Build.ArtifactStagingDirectory)/ - workingDirectory: $(WORK_DIR) - displayName: "Save coverage report" + # ------------------------------------------------------- + # Coverage report consolidation + - script: | + echo 'mode: count' > ./blobfuse2_coverage_raw.rpt + tail -q -n +2 ./*.cov >> ./blobfuse2_coverage_raw.rpt + cat ./blobfuse2_coverage_raw.rpt | grep -v mock_component | grep -v base_component | grep -v loopback | grep -v tools | grep -v "common/log" | grep -v "common/exectime" | grep -v "common/types.go" | grep -v "internal/stats_manager" | grep -v "main.go" | grep -v "component/azstorage/azauthmsi.go" | grep -v "component/azstorage/azauthspn.go" | grep -v "component/stream" | grep -v "component/custom" | grep -v "component/azstorage/azauthcli.go" | grep -v "exported/exported.go" | grep -v "component/block_cache/stream.go" | grep -v "component/azstorage/azAuthWorkloadIdentity.go" > ./blobfuse2_coverage.rpt + go tool cover -func blobfuse2_coverage.rpt > ./blobfuse2_func_cover.rpt + go tool cover -html=./blobfuse2_coverage.rpt -o ./blobfuse2_coverage.html + go tool cover -html=./blobfuse2_ut.cov -o ./blobfuse2_ut.html + cp ./*.rpt $(Build.ArtifactStagingDirectory)/ + cp ./*.cov $(Build.ArtifactStagingDirectory)/ + cp ./*.html $(Build.ArtifactStagingDirectory)/ + workingDirectory: $(WORK_DIR) + displayName: "Save coverage report" # Publish the Artifacts - task: PublishBuildArtifacts@1