From 43c29191e3d811ebeb711f102e8fad64af35de1a Mon Sep 17 00:00:00 2001 From: talbpaul Date: Tue, 31 Mar 2020 09:50:43 -0600 Subject: [PATCH 01/18] extraneous tracked file? --- .../DatabaseStorage/testGridRavenDatabase.h5 | Bin 46928 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 diff --git a/tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 b/tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 deleted file mode 100644 index 3bc11bac241fd73ba2d783daaa5ec692fbfefac3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 46928 zcmeHQ3v?9K8J>VZSONqMZ(9RZ6VxO#`^pAwCkP4IDj_A1qF|FvvXJ%Uv2HdJPK!d- z7OW4n538k?s;8|l)V`#)w%doVqo-%f!PcYIrdF+w@?P=N0SQ%@*{lOKUpl5aa<=&3qvS2XaYaa-D`&(6Og=(*`s+y}pb5y9@jy&Dn zD+7H4{w7bbs~?zfysozsqtGHJAG7_IPoWakQMTXRMKV1R^LUbJAin)_``anUHCBfv zPpp#KB(+QGkklzD_k&Uyk0;HIt5*~#6BRk!Dy0)C1?Ov!@h&MxgJe*=FP>9uoaGPS z5PSVZjm?sFD_&nkN)y;j94DqKXM>gleSN`RPmh=NFb~Ej)O9gb0O`C?sb6++a~)h; zyJj5{af9oFX*^$VZ>JciM0z+R%a`*NDGgFyi&6seZpOi{(GYlkL4kNu^a5C3%*%2u zf%7)$BG9#1e*x2CV14bRsKWk$@m#Y+`QAZ4jMLuERo z1Dme#Tr&h;9>;lHABo>4C~v5XNHv2`p zfTjVVRvaOl2-3X3dAYnYI#QsVD(Y&UOwRt+3)AVsPH8Vk^0)kpC_TwGy0XnwCyoZ< z?G?8tm-J)!Nwrq&weH?veRrQHXp9rJoFGX`_G)s(UWKO;1MU|`WPF~KgGZkuXU3*0 zJ&WSCWkgy2ur#pDFC-1>5qZ6ok}Jw*>4ejj%Xq^jbe((UP+M%eLXW)PK~yE89QtBx zx_Q@7+;yEykYU38e*a=(Tr2xMkIV6TG{)uA#CV6b+KPs(AL@1!)h%)wY25oUq~URS zGMpQHOANmKT^@fN;i_IWJg7PkzR=*q`@%K^P>a8^@WQ7hMYx0 zPU_zV%aQoahjzyNu#2u-zWQRJ1KCkoALvJ5NOry^o3XN#vMc4?e$CQSW%0CBjhdCk zZ?$+TiGg%~l?$0)r{I?qFSn{%`auRkC13OjF);t~Zn}ZK3I_CFFp&IBfxdQ6yRX|9 z-1=FsZ*x~rh}bgha>d#$7QYQ1)*tY0@(o)2b~tb9^YsR!l3M%@yc*B4V#PpD&sNRi zcjCii-^76Z6+;3!BAGyAEqK$>bu8}#*fEq)gY zIZX5#xXOZRDkd5B99Wd5$r@xo3C9Nv;yKFQnk9&pT<*52a0VKaXCNxw1X<|>v?1?- zXz~n1qnjWNouDZ4j7ny|2*(c$<)UCWB!jX&ZYv(_;s9c}?I3~Mi3gWBfEaEUiqH*G zCX>PK0(pbbl%yF{a9qGZ_1Em_@ppRzYq$DA!B}M-fxay(eEra#pxhR^-Pz&^^i!>I zGs2qdgk)l|#Mjr069THZ=&QsQYcPsYQ@f~2+3XcsNK_OYS1`oa2UZM-Lb^;)8<<8^ z2A)7&An$2{TNK|oeS<<#v#3~j7&q3cq1)j%-M}B!2;&_E9Jer#9#;l@okD@-ogl4f z+;&TI+2n4r)2PUNnAdA&Q1emF zk#zLdh%(HS`MM>gc9?=)OTrFTfiZwiBOZs1Qnl0tZv|XAL&o@JMX|Cj3X4<6;+oerO+a+tn zE(uQn%u%#)0_zYl-C%(d8mTA~BsfH@6!?l@dcbz!iLg@Eh?R0x+g+N?>2%nu9W|=G z+CGw%;&l$5XYoG5eA%8-wNDFV{yQbjqJ2ujo>W@#3!k&Kx6$Y zYF#n1{x|yds_6AEoMRe%X+ICv{|shuKmGN;MEh>X-|xcq%j;@52Z@(6-Tgt)nFL?n zADk)H|JE#PscUTYdjj6xpnC)67qUb zOd)3Egx5f#eX=&$j#vTK4DXY1dwl_X9SldL)#Lr0`28u>1y{No+f+9oByi+z!nN>sMd?i{Mx1h4tm9zw@edL0F&i`}v=FabZ}0?&^kL zZTnbQ->q$_Sae31CAANa~kT?)`k=05rdq?7Kxb!h?6AEMYmKlhlsXadk_^=Azr zg;cK~?Lrn+_Re}0TprsU0C(S??GK_Ae%fv2xr z4Rp!%uj>w=)4wxe>O!E0essnG1?a)2FZk9Q`hw8@moMpj4(Oi4S62KG=+#rc_Wo@^ zJ5GP(SKEND)PBFS1L(Q+?|jA$bjF6lg;hWg_nq}x3D7@Yd*Op0eriGJvD;g(90L07 zeP7U@1iIt5w--JDla$o; zt7N(+y-Fx;-^}fU68?p8O_|*XWoymtBYX3M zVhN?ould3D^H@_RO-2L8K6d*1?YO3C^s~(Ex8s_m!8dALNOK;8Ym&zE(>^}=ABN*R z(mp<}JF*{-YmtVWxjQdHUp`*WT$~r-TAbl`WqMvS_n7k*#tdPhgc&x{%^Aul&_R(q9EC{8gaB zUj-`sRiMIO1uFbipu%4TD*RQT!e0d{{8gaBUj-`sRiMIO1uEPwpu%4TN`9L3S0OI^ zRiMIO1uE1Jbk3yzZg0HtnTVfd`m3q=tA5wJA6-%Aj_7ei{RNBeI`p1C()vW31WK9f z6DbY|d7jO%hnbtt;NFD6H_C5GaXu6JBqFDP(XTu=C z{1E$U$eDZd8J3gwFnEUogg!<{gs;vv?nt`vdM~$|xbc6i-#?il%gr@=%Q*UoUBvgp=xr16 zK2WOfpD;c?!Z%Le!(TCh1pIE#@SBnMPZ;lR+S{$!!}%t#xAyU~w~2AS3G6NI{bFEy zYp1Bf`2SSxE$($;Aao`BexCephL@NokKe=t+D?saBx?{`lfG#=oXzhJKr%!r*>TaO_n1A@d0MLV#Gxux& zx_|k3-eo}dtan$I0o~rS>4PGm9bddAbXYA3t-NjTt`~rw`@rLCehGBuuU}kv575KU zfAHEapnrV3WM3c9$Bt@S)&aehk@LGY-DJKGI%!f0g^0Nlgvpq6Xsk zF>(p-#QPYCJuvW2so~6x{F%l}9s7G}A^DxBA?fdrgtK^fBef0_XOWA&Wb9>1gs1D0 z?T96?j}e~k$YoT*n{wZV@pSRuEi=Z8^zk6vzhmEMx`9?{>=tK-0e?y`5~w zME+faY)7mQ#~Ar{FKng~3gvu+-+vg|PS=a${J!|_&>G`N=KMZ-&xqd>j2iFL^!sAp zH%i~{I^%Qy5GnAQ02zFG_`Rrd;Z;!{P0PB`FGpB)V|GQavCuDduHb&2H*7k zM(#(kc;u*YBt^fG`6Bi*5k7Xja5Hk4&bjZlGJZCKZ^;mDW*Gi8L%5kqvOCO{3KeWy zOq}YN;#A@5a{x-*;T6w&{U@lKBy}`Y|40M={6}2*=`uGnkr6) z=K*%fnA+b-jFU+Q3*!*q0k6cWV0>>A>+!_K)r=ZXQw2IgM2*1kq=3e$mS`Nt7w0<; zhw;MM#^Er&cf9-s#`*4$;}(rG1;7h@V-6?+2#){lULJ@ygrx_}55% z7Kff%vh>zxkIoF;+#EdfHlQur_WiUS=*PeRW=*Tain`p^Gnh93NIW6u*nG1YFM z7c~CimW@E+fBX2*hfBEso9#8#?`4JpcdskHZDXXpn)p~AVN9CK8Zh?z=({1bKRbHd2+kLI zKLOXAjs1_@!;P?Tnq_UV?oB@(6Lcc}v$0cq>f8?kH{6kNDnN z0f^4e%haDhlH%{h-jM1t9x(Q{88bCtYQWThsR2_1rUp&`4J5)#pDUkj^Z+)y$iaB$ zMuOU~0hBfT;mf1EvN}Y7O8#v-lgN0-Sr|yp`iPmyM3&+&Vgr zbLHqb?tw(daV;P^F6Q>IF~fT^aN7)~222f@8Zb3rYQWUMIA~xb{Mm5)M7aP);O}^& zy)Op%1HEcjsMIgJxVa9ltzEMYiMVkbJVoh4B9ZthF=7Gn;dqKJN|mK!BK!oe%g2kc eJpq0q@<8z!%($rmQv;?3ObwVCFg1|(8u&l5#ivvN From 702c2f4a2e39d446e81b57c9c3724e386c607426 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Wed, 13 May 2020 08:53:08 -0600 Subject: [PATCH 02/18] heron as a submodule --- .gitmodules | 3 +++ plugins/heron | 1 + 2 files changed, 4 insertions(+) create mode 160000 plugins/heron diff --git a/.gitmodules b/.gitmodules index 56c51e3157..52869f2bfd 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,6 @@ [submodule "plugins/SRAW"] path = plugins/SRAW url = git@hpcgitlab.hpc.inl.gov:RAVEN_PLUGINS/SRAW.git +[submodule "plugins/heron"] + path = plugins/heron + url = git@hpcgitlab.hpc.inl.gov:RAVEN_PLUGINS/heron.git diff --git a/plugins/heron b/plugins/heron new file mode 160000 index 0000000000..6697715683 --- /dev/null +++ b/plugins/heron @@ -0,0 +1 @@ +Subproject commit 669771568399bb8fc9645abd11e0323fc5d31cd2 From de3e8105e0948e0714c7c6e1143f5785c14ea154 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Wed, 13 May 2020 08:55:15 -0600 Subject: [PATCH 03/18] undoing file removal --- .../DatabaseStorage/testGridRavenDatabase.h5 | Bin 0 -> 46928 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 diff --git a/tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 b/tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 new file mode 100644 index 0000000000000000000000000000000000000000..3bc11bac241fd73ba2d783daaa5ec692fbfefac3 GIT binary patch literal 46928 zcmeHQ3v?9K8J>VZSONqMZ(9RZ6VxO#`^pAwCkP4IDj_A1qF|FvvXJ%Uv2HdJPK!d- z7OW4n538k?s;8|l)V`#)w%doVqo-%f!PcYIrdF+w@?P=N0SQ%@*{lOKUpl5aa<=&3qvS2XaYaa-D`&(6Og=(*`s+y}pb5y9@jy&Dn zD+7H4{w7bbs~?zfysozsqtGHJAG7_IPoWakQMTXRMKV1R^LUbJAin)_``anUHCBfv zPpp#KB(+QGkklzD_k&Uyk0;HIt5*~#6BRk!Dy0)C1?Ov!@h&MxgJe*=FP>9uoaGPS z5PSVZjm?sFD_&nkN)y;j94DqKXM>gleSN`RPmh=NFb~Ej)O9gb0O`C?sb6++a~)h; zyJj5{af9oFX*^$VZ>JciM0z+R%a`*NDGgFyi&6seZpOi{(GYlkL4kNu^a5C3%*%2u zf%7)$BG9#1e*x2CV14bRsKWk$@m#Y+`QAZ4jMLuERo z1Dme#Tr&h;9>;lHABo>4C~v5XNHv2`p zfTjVVRvaOl2-3X3dAYnYI#QsVD(Y&UOwRt+3)AVsPH8Vk^0)kpC_TwGy0XnwCyoZ< z?G?8tm-J)!Nwrq&weH?veRrQHXp9rJoFGX`_G)s(UWKO;1MU|`WPF~KgGZkuXU3*0 zJ&WSCWkgy2ur#pDFC-1>5qZ6ok}Jw*>4ejj%Xq^jbe((UP+M%eLXW)PK~yE89QtBx zx_Q@7+;yEykYU38e*a=(Tr2xMkIV6TG{)uA#CV6b+KPs(AL@1!)h%)wY25oUq~URS zGMpQHOANmKT^@fN;i_IWJg7PkzR=*q`@%K^P>a8^@WQ7hMYx0 zPU_zV%aQoahjzyNu#2u-zWQRJ1KCkoALvJ5NOry^o3XN#vMc4?e$CQSW%0CBjhdCk zZ?$+TiGg%~l?$0)r{I?qFSn{%`auRkC13OjF);t~Zn}ZK3I_CFFp&IBfxdQ6yRX|9 z-1=FsZ*x~rh}bgha>d#$7QYQ1)*tY0@(o)2b~tb9^YsR!l3M%@yc*B4V#PpD&sNRi zcjCii-^76Z6+;3!BAGyAEqK$>bu8}#*fEq)gY zIZX5#xXOZRDkd5B99Wd5$r@xo3C9Nv;yKFQnk9&pT<*52a0VKaXCNxw1X<|>v?1?- zXz~n1qnjWNouDZ4j7ny|2*(c$<)UCWB!jX&ZYv(_;s9c}?I3~Mi3gWBfEaEUiqH*G zCX>PK0(pbbl%yF{a9qGZ_1Em_@ppRzYq$DA!B}M-fxay(eEra#pxhR^-Pz&^^i!>I zGs2qdgk)l|#Mjr069THZ=&QsQYcPsYQ@f~2+3XcsNK_OYS1`oa2UZM-Lb^;)8<<8^ z2A)7&An$2{TNK|oeS<<#v#3~j7&q3cq1)j%-M}B!2;&_E9Jer#9#;l@okD@-ogl4f z+;&TI+2n4r)2PUNnAdA&Q1emF zk#zLdh%(HS`MM>gc9?=)OTrFTfiZwiBOZs1Qnl0tZv|XAL&o@JMX|Cj3X4<6;+oerO+a+tn zE(uQn%u%#)0_zYl-C%(d8mTA~BsfH@6!?l@dcbz!iLg@Eh?R0x+g+N?>2%nu9W|=G z+CGw%;&l$5XYoG5eA%8-wNDFV{yQbjqJ2ujo>W@#3!k&Kx6$Y zYF#n1{x|yds_6AEoMRe%X+ICv{|shuKmGN;MEh>X-|xcq%j;@52Z@(6-Tgt)nFL?n zADk)H|JE#PscUTYdjj6xpnC)67qUb zOd)3Egx5f#eX=&$j#vTK4DXY1dwl_X9SldL)#Lr0`28u>1y{No+f+9oByi+z!nN>sMd?i{Mx1h4tm9zw@edL0F&i`}v=FabZ}0?&^kL zZTnbQ->q$_Sae31CAANa~kT?)`k=05rdq?7Kxb!h?6AEMYmKlhlsXadk_^=Azr zg;cK~?Lrn+_Re}0TprsU0C(S??GK_Ae%fv2xr z4Rp!%uj>w=)4wxe>O!E0essnG1?a)2FZk9Q`hw8@moMpj4(Oi4S62KG=+#rc_Wo@^ zJ5GP(SKEND)PBFS1L(Q+?|jA$bjF6lg;hWg_nq}x3D7@Yd*Op0eriGJvD;g(90L07 zeP7U@1iIt5w--JDla$o; zt7N(+y-Fx;-^}fU68?p8O_|*XWoymtBYX3M zVhN?ould3D^H@_RO-2L8K6d*1?YO3C^s~(Ex8s_m!8dALNOK;8Ym&zE(>^}=ABN*R z(mp<}JF*{-YmtVWxjQdHUp`*WT$~r-TAbl`WqMvS_n7k*#tdPhgc&x{%^Aul&_R(q9EC{8gaB zUj-`sRiMIO1uFbipu%4TD*RQT!e0d{{8gaBUj-`sRiMIO1uEPwpu%4TN`9L3S0OI^ zRiMIO1uE1Jbk3yzZg0HtnTVfd`m3q=tA5wJA6-%Aj_7ei{RNBeI`p1C()vW31WK9f z6DbY|d7jO%hnbtt;NFD6H_C5GaXu6JBqFDP(XTu=C z{1E$U$eDZd8J3gwFnEUogg!<{gs;vv?nt`vdM~$|xbc6i-#?il%gr@=%Q*UoUBvgp=xr16 zK2WOfpD;c?!Z%Le!(TCh1pIE#@SBnMPZ;lR+S{$!!}%t#xAyU~w~2AS3G6NI{bFEy zYp1Bf`2SSxE$($;Aao`BexCephL@NokKe=t+D?saBx?{`lfG#=oXzhJKr%!r*>TaO_n1A@d0MLV#Gxux& zx_|k3-eo}dtan$I0o~rS>4PGm9bddAbXYA3t-NjTt`~rw`@rLCehGBuuU}kv575KU zfAHEapnrV3WM3c9$Bt@S)&aehk@LGY-DJKGI%!f0g^0Nlgvpq6Xsk zF>(p-#QPYCJuvW2so~6x{F%l}9s7G}A^DxBA?fdrgtK^fBef0_XOWA&Wb9>1gs1D0 z?T96?j}e~k$YoT*n{wZV@pSRuEi=Z8^zk6vzhmEMx`9?{>=tK-0e?y`5~w zME+faY)7mQ#~Ar{FKng~3gvu+-+vg|PS=a${J!|_&>G`N=KMZ-&xqd>j2iFL^!sAp zH%i~{I^%Qy5GnAQ02zFG_`Rrd;Z;!{P0PB`FGpB)V|GQavCuDduHb&2H*7k zM(#(kc;u*YBt^fG`6Bi*5k7Xja5Hk4&bjZlGJZCKZ^;mDW*Gi8L%5kqvOCO{3KeWy zOq}YN;#A@5a{x-*;T6w&{U@lKBy}`Y|40M={6}2*=`uGnkr6) z=K*%fnA+b-jFU+Q3*!*q0k6cWV0>>A>+!_K)r=ZXQw2IgM2*1kq=3e$mS`Nt7w0<; zhw;MM#^Er&cf9-s#`*4$;}(rG1;7h@V-6?+2#){lULJ@ygrx_}55% z7Kff%vh>zxkIoF;+#EdfHlQur_WiUS=*PeRW=*Tain`p^Gnh93NIW6u*nG1YFM z7c~CimW@E+fBX2*hfBEso9#8#?`4JpcdskHZDXXpn)p~AVN9CK8Zh?z=({1bKRbHd2+kLI zKLOXAjs1_@!;P?Tnq_UV?oB@(6Lcc}v$0cq>f8?kH{6kNDnN z0f^4e%haDhlH%{h-jM1t9x(Q{88bCtYQWThsR2_1rUp&`4J5)#pDUkj^Z+)y$iaB$ zMuOU~0hBfT;mf1EvN}Y7O8#v-lgN0-Sr|yp`iPmyM3&+&Vgr zbLHqb?tw(daV;P^F6Q>IF~fT^aN7)~222f@8Zb3rYQWUMIA~xb{Mm5)M7aP);O}^& zy)Op%1HEcjsMIgJxVa9ltzEMYiMVkbJVoh4B9ZthF=7Gn;dqKJN|mK!BK!oe%g2kc eJpq0q@<8z!%($rmQv;?3ObwVCFg1|(8u&l5#ivvN literal 0 HcmV?d00001 From dc8c91a9982bd0a3136f993bf3d7579c7104a1e7 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Wed, 13 May 2020 10:20:25 -0600 Subject: [PATCH 04/18] update with class docs --- plugins/heron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/heron b/plugins/heron index 6697715683..a6f4d1d871 160000 --- a/plugins/heron +++ b/plugins/heron @@ -1 +1 @@ -Subproject commit 669771568399bb8fc9645abd11e0323fc5d31cd2 +Subproject commit a6f4d1d87129b1923a52fb624ac4f4ee948d4f12 From 148adb6c967131a84be4e5b0706608bad0d63e97 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Thu, 25 Jun 2020 11:50:31 -0600 Subject: [PATCH 05/18] matching remote devel --- .../DatabaseStorage/testGridRavenDatabase.h5 | Bin 0 -> 46928 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 diff --git a/tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 b/tests/framework/DataObjects/DataobjectsAttributes/DatabaseStorage/testGridRavenDatabase.h5 new file mode 100644 index 0000000000000000000000000000000000000000..3bc11bac241fd73ba2d783daaa5ec692fbfefac3 GIT binary patch literal 46928 zcmeHQ3v?9K8J>VZSONqMZ(9RZ6VxO#`^pAwCkP4IDj_A1qF|FvvXJ%Uv2HdJPK!d- z7OW4n538k?s;8|l)V`#)w%doVqo-%f!PcYIrdF+w@?P=N0SQ%@*{lOKUpl5aa<=&3qvS2XaYaa-D`&(6Og=(*`s+y}pb5y9@jy&Dn zD+7H4{w7bbs~?zfysozsqtGHJAG7_IPoWakQMTXRMKV1R^LUbJAin)_``anUHCBfv zPpp#KB(+QGkklzD_k&Uyk0;HIt5*~#6BRk!Dy0)C1?Ov!@h&MxgJe*=FP>9uoaGPS z5PSVZjm?sFD_&nkN)y;j94DqKXM>gleSN`RPmh=NFb~Ej)O9gb0O`C?sb6++a~)h; zyJj5{af9oFX*^$VZ>JciM0z+R%a`*NDGgFyi&6seZpOi{(GYlkL4kNu^a5C3%*%2u zf%7)$BG9#1e*x2CV14bRsKWk$@m#Y+`QAZ4jMLuERo z1Dme#Tr&h;9>;lHABo>4C~v5XNHv2`p zfTjVVRvaOl2-3X3dAYnYI#QsVD(Y&UOwRt+3)AVsPH8Vk^0)kpC_TwGy0XnwCyoZ< z?G?8tm-J)!Nwrq&weH?veRrQHXp9rJoFGX`_G)s(UWKO;1MU|`WPF~KgGZkuXU3*0 zJ&WSCWkgy2ur#pDFC-1>5qZ6ok}Jw*>4ejj%Xq^jbe((UP+M%eLXW)PK~yE89QtBx zx_Q@7+;yEykYU38e*a=(Tr2xMkIV6TG{)uA#CV6b+KPs(AL@1!)h%)wY25oUq~URS zGMpQHOANmKT^@fN;i_IWJg7PkzR=*q`@%K^P>a8^@WQ7hMYx0 zPU_zV%aQoahjzyNu#2u-zWQRJ1KCkoALvJ5NOry^o3XN#vMc4?e$CQSW%0CBjhdCk zZ?$+TiGg%~l?$0)r{I?qFSn{%`auRkC13OjF);t~Zn}ZK3I_CFFp&IBfxdQ6yRX|9 z-1=FsZ*x~rh}bgha>d#$7QYQ1)*tY0@(o)2b~tb9^YsR!l3M%@yc*B4V#PpD&sNRi zcjCii-^76Z6+;3!BAGyAEqK$>bu8}#*fEq)gY zIZX5#xXOZRDkd5B99Wd5$r@xo3C9Nv;yKFQnk9&pT<*52a0VKaXCNxw1X<|>v?1?- zXz~n1qnjWNouDZ4j7ny|2*(c$<)UCWB!jX&ZYv(_;s9c}?I3~Mi3gWBfEaEUiqH*G zCX>PK0(pbbl%yF{a9qGZ_1Em_@ppRzYq$DA!B}M-fxay(eEra#pxhR^-Pz&^^i!>I zGs2qdgk)l|#Mjr069THZ=&QsQYcPsYQ@f~2+3XcsNK_OYS1`oa2UZM-Lb^;)8<<8^ z2A)7&An$2{TNK|oeS<<#v#3~j7&q3cq1)j%-M}B!2;&_E9Jer#9#;l@okD@-ogl4f z+;&TI+2n4r)2PUNnAdA&Q1emF zk#zLdh%(HS`MM>gc9?=)OTrFTfiZwiBOZs1Qnl0tZv|XAL&o@JMX|Cj3X4<6;+oerO+a+tn zE(uQn%u%#)0_zYl-C%(d8mTA~BsfH@6!?l@dcbz!iLg@Eh?R0x+g+N?>2%nu9W|=G z+CGw%;&l$5XYoG5eA%8-wNDFV{yQbjqJ2ujo>W@#3!k&Kx6$Y zYF#n1{x|yds_6AEoMRe%X+ICv{|shuKmGN;MEh>X-|xcq%j;@52Z@(6-Tgt)nFL?n zADk)H|JE#PscUTYdjj6xpnC)67qUb zOd)3Egx5f#eX=&$j#vTK4DXY1dwl_X9SldL)#Lr0`28u>1y{No+f+9oByi+z!nN>sMd?i{Mx1h4tm9zw@edL0F&i`}v=FabZ}0?&^kL zZTnbQ->q$_Sae31CAANa~kT?)`k=05rdq?7Kxb!h?6AEMYmKlhlsXadk_^=Azr zg;cK~?Lrn+_Re}0TprsU0C(S??GK_Ae%fv2xr z4Rp!%uj>w=)4wxe>O!E0essnG1?a)2FZk9Q`hw8@moMpj4(Oi4S62KG=+#rc_Wo@^ zJ5GP(SKEND)PBFS1L(Q+?|jA$bjF6lg;hWg_nq}x3D7@Yd*Op0eriGJvD;g(90L07 zeP7U@1iIt5w--JDla$o; zt7N(+y-Fx;-^}fU68?p8O_|*XWoymtBYX3M zVhN?ould3D^H@_RO-2L8K6d*1?YO3C^s~(Ex8s_m!8dALNOK;8Ym&zE(>^}=ABN*R z(mp<}JF*{-YmtVWxjQdHUp`*WT$~r-TAbl`WqMvS_n7k*#tdPhgc&x{%^Aul&_R(q9EC{8gaB zUj-`sRiMIO1uFbipu%4TD*RQT!e0d{{8gaBUj-`sRiMIO1uEPwpu%4TN`9L3S0OI^ zRiMIO1uE1Jbk3yzZg0HtnTVfd`m3q=tA5wJA6-%Aj_7ei{RNBeI`p1C()vW31WK9f z6DbY|d7jO%hnbtt;NFD6H_C5GaXu6JBqFDP(XTu=C z{1E$U$eDZd8J3gwFnEUogg!<{gs;vv?nt`vdM~$|xbc6i-#?il%gr@=%Q*UoUBvgp=xr16 zK2WOfpD;c?!Z%Le!(TCh1pIE#@SBnMPZ;lR+S{$!!}%t#xAyU~w~2AS3G6NI{bFEy zYp1Bf`2SSxE$($;Aao`BexCephL@NokKe=t+D?saBx?{`lfG#=oXzhJKr%!r*>TaO_n1A@d0MLV#Gxux& zx_|k3-eo}dtan$I0o~rS>4PGm9bddAbXYA3t-NjTt`~rw`@rLCehGBuuU}kv575KU zfAHEapnrV3WM3c9$Bt@S)&aehk@LGY-DJKGI%!f0g^0Nlgvpq6Xsk zF>(p-#QPYCJuvW2so~6x{F%l}9s7G}A^DxBA?fdrgtK^fBef0_XOWA&Wb9>1gs1D0 z?T96?j}e~k$YoT*n{wZV@pSRuEi=Z8^zk6vzhmEMx`9?{>=tK-0e?y`5~w zME+faY)7mQ#~Ar{FKng~3gvu+-+vg|PS=a${J!|_&>G`N=KMZ-&xqd>j2iFL^!sAp zH%i~{I^%Qy5GnAQ02zFG_`Rrd;Z;!{P0PB`FGpB)V|GQavCuDduHb&2H*7k zM(#(kc;u*YBt^fG`6Bi*5k7Xja5Hk4&bjZlGJZCKZ^;mDW*Gi8L%5kqvOCO{3KeWy zOq}YN;#A@5a{x-*;T6w&{U@lKBy}`Y|40M={6}2*=`uGnkr6) z=K*%fnA+b-jFU+Q3*!*q0k6cWV0>>A>+!_K)r=ZXQw2IgM2*1kq=3e$mS`Nt7w0<; zhw;MM#^Er&cf9-s#`*4$;}(rG1;7h@V-6?+2#){lULJ@ygrx_}55% z7Kff%vh>zxkIoF;+#EdfHlQur_WiUS=*PeRW=*Tain`p^Gnh93NIW6u*nG1YFM z7c~CimW@E+fBX2*hfBEso9#8#?`4JpcdskHZDXXpn)p~AVN9CK8Zh?z=({1bKRbHd2+kLI zKLOXAjs1_@!;P?Tnq_UV?oB@(6Lcc}v$0cq>f8?kH{6kNDnN z0f^4e%haDhlH%{h-jM1t9x(Q{88bCtYQWThsR2_1rUp&`4J5)#pDUkj^Z+)y$iaB$ zMuOU~0hBfT;mf1EvN}Y7O8#v-lgN0-Sr|yp`iPmyM3&+&Vgr zbLHqb?tw(daV;P^F6Q>IF~fT^aN7)~222f@8Zb3rYQWUMIA~xb{Mm5)M7aP);O}^& zy)Op%1HEcjsMIgJxVa9ltzEMYiMVkbJVoh4B9ZthF=7Gn;dqKJN|mK!BK!oe%g2kc eJpq0q@<8z!%($rmQv;?3ObwVCFg1|(8u&l5#ivvN literal 0 HcmV?d00001 From 2cd7ca0655986511dd153fedfd811bd8bfd124ae Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Fri, 10 Jul 2020 10:51:10 -0600 Subject: [PATCH 06/18] updated to latest heron --- plugins/heron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/heron b/plugins/heron index a6f4d1d871..ceab3f4aa4 160000 --- a/plugins/heron +++ b/plugins/heron @@ -1 +1 @@ -Subproject commit a6f4d1d87129b1923a52fb624ac4f4ee948d4f12 +Subproject commit ceab3f4aa437d6d89f28d10c9a0636190a8b6913 From 353ac6c251961fbbd4ae902d761727c082a490b1 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 13 Jul 2020 11:18:03 -0600 Subject: [PATCH 07/18] fixed submodule --- plugins/heron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/heron b/plugins/heron index ceab3f4aa4..da96742e38 160000 --- a/plugins/heron +++ b/plugins/heron @@ -1 +1 @@ -Subproject commit ceab3f4aa437d6d89f28d10c9a0636190a8b6913 +Subproject commit da96742e38d5e9626a7c9cc77569fec330ed0bd3 From 4e152401fbdb6ada0627f7103b92a176a2c3f330 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 13 Jul 2020 13:00:26 -0600 Subject: [PATCH 08/18] TESTING build --- plugins/heron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/heron b/plugins/heron index da96742e38..55921ac420 160000 --- a/plugins/heron +++ b/plugins/heron @@ -1 +1 @@ -Subproject commit da96742e38d5e9626a7c9cc77569fec330ed0bd3 +Subproject commit 55921ac420fc360a3bcd2c12f982be921268242e From 31cac93284be400c002b539fe72b732d808ed117 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 13 Jul 2020 13:07:52 -0600 Subject: [PATCH 09/18] did not work, trying again --- plugins/heron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/heron b/plugins/heron index 55921ac420..1c78c5a823 160000 --- a/plugins/heron +++ b/plugins/heron @@ -1 +1 @@ -Subproject commit 55921ac420fc360a3bcd2c12f982be921268242e +Subproject commit 1c78c5a8230661e48ec84c1edc43aabc065af902 From a4551c75693452e1348f5e1cb08601e9ac4417f1 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Thu, 16 Jul 2020 16:22:03 -0600 Subject: [PATCH 10/18] wip --- dependencies.ini | 100 -------------------------- dependencies.xml | 67 ++++++++++++++++++ scripts/library_handler.py | 141 ++++++++++++++++++++++++++++++++----- 3 files changed, 192 insertions(+), 116 deletions(-) delete mode 100644 dependencies.ini create mode 100644 dependencies.xml diff --git a/dependencies.ini b/dependencies.ini deleted file mode 100644 index b3c3f3c1e7..0000000000 --- a/dependencies.ini +++ /dev/null @@ -1,100 +0,0 @@ -# Instructions: -# -# add library: library = pinned.version.number -# add with no pinned version: library -# remove a library: library = remove -# -# end instructions - -# Reading flow: -# -# The library handler will proceed in the following pattern for reading this file: -# 1. Read in "core" (stuff that comes from conda) -# 2. Read in "forge" (stuff that comes from conda-forge) -# 3. Read in "pip" (stuff that comes from pip) (NOTE: not the pip-only install!) -# 4. If optional, then read in "optional" -# 5. If OS is given, read in either "windows", "mac", or "linux" -# 6. If pip-only install, then read in "pip-install" -# -# Note that each reading-in stage might involve adding or removing libraries, in order. -# If a library is not present but requested to remove, that's fine, no crash, move on. -# -# For more information about the command line arguments to library_handler.sh, see that module. -# Also try -# `python library_handler.sh -h` and -# `python library_handler.sh conda -h` -# -# end reading flow - -##### Typical installation (conda) -# main libraries from main conda -[core] -h5py = 2.9.0 -numpy = 1.18.1 -scipy = 1.2.1 -scikit-learn = 0.21.2 -pandas = 0.24.2 -xarray = 0.12.1 -netcdf4 = 1.4.2 -matplotlib = 3.1.1 -statsmodels = 0.9.0 -cloudpickle = 1.1.1 -tensorflow = 1.13.1 -python = 3 -hdf5 -swig -pylint -coverage -lxml -psutil -pip -importlib_metadata - -# secondary installs from conda forge -[forge] -pyside2 - -# tertiary installs directly from pip -[pip] - -# optional libraries -[optional] -pillow = 6.0.0 - -# operating system alterations (conda) -[windows] -# nothing currently - -[mac] -# nothing currently - -[linux] -# these prevent Intel crash errors with mkl -nomkl -numexpr - -# operating system alterations (pip) -[windows-pip] -# nothing currently - -[mac-pip] -ray - -[linux-pip] -ray - -##### Alternate installs -# install using pip -[pip-install] -hdf5 = remove -swig = remove -pip = remove -python = remove -nomkl = remove -numexpr = remove - -[skip-check] -python -hdf5 -swig -nomkl diff --git a/dependencies.xml b/dependencies.xml new file mode 100644 index 0000000000..0f6da82002 --- /dev/null +++ b/dependencies.xml @@ -0,0 +1,67 @@ + + +
+ 2.9.0 + 1.18.1 + 1.2.1 + 0.21.2 + 0.24.2 + 0.12.1 + 1.4.2 + 3.1.1 + 0.9.0 + 1.9.1 + 1.13.1 + 3 + + + + + + + + + + 6.0.0 + + + +
+ + remove + remove + remove + remove + remove + remove + + diff --git a/scripts/library_handler.py b/scripts/library_handler.py index 386e764c4a..2f9f7ec6e1 100644 --- a/scripts/library_handler.py +++ b/scripts/library_handler.py @@ -21,6 +21,8 @@ import argparse import subprocess from collections import OrderedDict +import xml.etree.ElementTree as ET +# NOTE: DO NOT use xmlUtils here! we need to avoid importing any non-standard Python library! from update_install_data import loadRC import plugin_handler as pluginHandler @@ -31,12 +33,6 @@ else: impErr = ImportError -# python 2.X uses a different capitalization for configparser -try: - import configparser -except impErr: - import ConfigParser as configparser - try: # python 3.8+ includes this in std lib import importlib_metadata @@ -225,15 +221,24 @@ def getRequiredLibs(useOS=None, installMethod=None, addOptional=False, limit=Non the required libs @ Out, libs, dict, dictionary of libraries {name: version} """ - mainConfigFile = os.path.abspath(os.path.expanduser(os.path.join(os.path.dirname(__file__), - '..', 'dependencies.ini'))) - config = _readDependencies(mainConfigFile) + # OLD config = _readDependencies(mainConfigFile) opSys = _getOperatingSystem(override=useOS) install = _getInstallMethod(override=installMethod) + mainConfigFile = os.path.abspath(os.path.expanduser(os.path.join(os.path.dirname(__file__), + '..', 'dependencies.xml'))) + sourceFiles = [mainConfigFile] + for pluginName, pluginLoc in plugins: + pluginConfigFile = os.path.join(pluginLoc, 'dependencies.xml') + if os.path.isfile(pluginConfigFile): + sourceFiles.append(pluginConfigFile) + config = _combineSources(sourceFiles, opSys, install, addOptional=addOptional, limit=limit) + libs = _parseLibs(config) #, opSys, install, addOptional=addOptional, limit=limit) + + #### OLD #### libs = _parseLibs(config, opSys, install, addOptional=addOptional, limit=limit) # extend config with plugin libs for pluginName, pluginLoc in plugins: - pluginConfigFile = os.path.join(pluginLoc, 'dependencies.ini') + pluginConfigFile = os.path.join(pluginLoc, 'dependencies.xml') if os.path.isfile(pluginConfigFile): pluginConfig = _readDependencies(pluginConfigFile) pluginLibs = _parseLibs(pluginConfig, opSys, install, addOptional=addOptional, limit=limit) @@ -250,7 +255,7 @@ def getSkipCheckLibs(plugins=None): """ skipCheckLibs = OrderedDict() mainConfigFile = os.path.abspath(os.path.expanduser(os.path.join(os.path.dirname(__file__), - '..', 'dependencies.ini'))) + '..', 'dependencies.xml'))) config = _readDependencies(mainConfigFile) if config.has_section('skip-check'): _addLibsFromSection(config.items('skip-check'), skipCheckLibs) @@ -349,7 +354,112 @@ def _getInstallMethod(override=None): # no suggestion given, so we assume conda return 'conda' -def _parseLibs(config, opSys, install, addOptional=False, limit=None, plugins=None): +def _combineSources(sources, opSys, install, addOptional=False, limit=None): + """ + Parses config file to get libraries to install, using given options. + @ In, sources, list(str), full-path dependency file locations + @ In, opSys, str, operating system (not checked) + @ In, install, str, installation method (not checked) + @ In, addOptional, bool, optional, if True then include optional libraries + @ In, limit, list(str), optional, if provided then only read the given sections + @ In, plugins, list(tuple(str,configParser.configParser)), optional, plugins (name, config) + that should be added to the parsing + @ Out, config, dict, dictionary of libraries {name: version} + """ + config = dict() + toRemove = [] + for source in sources: + src = _readDependencies(source) + # always load main, if present + root = src.find('main') + if root is not None: + for libNode in root: + tag = libNode.tag + # check OS + ## note that None means "mac,os,linux" in this case + libOS = libNode.attrib.get('os', None) + # does library have a specified OS? + if libOS is not None: + # if this library's OS's don't match the requested OS, then we move on + if opSys not in [x.lower().strip() for x in libOS.split(',')]: + continue + # check optional + ## note that None means "not optional" in this case + ## further note anything besides "True" is taken to mean "not optional" + libOptional = libNode.attrib.get('optional', None) + if libOptional.strip().lower() == 'true' and not addOptional: + continue + # otherwise, we have a valid request to handle + text = root.text + if text is not None: + text = text.strip().lower() + # check for removal + ## this says the library should be removed from the existing list, which we do at the end! + if text == 'remove': + toRemove.append(tag) + continue + else: + version = text + libSource = libNode.attrib.get('source', None) + libSkipCheck = libNode.attrib.get('skip_check', None) + request = {'source': libSource, 'skip_check': libSkipCheck, 'version': version} + # does this entry already exist? + if tag in config: + existing = config[tag] + # check if either existing or requested is default (None) for each of the dictionary entries + for requestEntry, requestValue in request.items(): + existingFlexible = existing['version'] in [None, ''] + requestFlexible = libVersion['version'] in [None, ''] + # if niether is flexible AND they conflict, we need to error! + if not existingFlexible and not requestFlexible and existing != libVersion: + raise IOError(f'Dependency "{tag}" has conflicting requirements ({existing} vs {text})! Note the conflict may come from a plugin.') + # if new request is flexible, take the pinned one + if requestFlexible: + config[tag] = existing + # otherwise, take the new request. + else: + config[tag] = text + # if doesn't already exist, just add it to the queue + else: + config[tag] = existing + + TODO + else: + # if using an alternate install (not main), search through alternates + for candidate in srs.findall(alternate): + if alternate.attrib['name'] == install: + root = candidate + break + else: + root = None + + +def _parseLibs(node, existing, opSys, install, addOptional=False, limit=None, plugins=None): + """ + Parses lib file to get libraries to install, using given options. + @ In, config, xml.etree.ElementTree.Element, read-in dependencies + @ In, opSys, str, operating system (not checked) + @ In, install, str, installation method (not checked) + @ In, addOptional, bool, optional, if True then include optional libraries + @ In, limit, list(str), optional, if provided then only read the given sections + @ In, plugins, list(tuple(str,configParser.configParser)), optional, plugins (name, config) + that should be added to the parsing + @ Out, libs, dict, dictionary of libraries {name: version} + """ + libs = OrderedDict() + # find all the applicable libs for this opSys, install method, optionals + if install == 'main': + root = config.find('main') + else: + for candidate in config.findall('alternate'): + if candidate.attrib['name'] == install: + root = candidate + break + else: + return None + + +def OLD_____parseLibs(config, opSys, install, addOptional=False, limit=None, plugins=None): """ Parses config file to get libraries to install, using given options. @ In, config, configparser.ConfigParser, read-in dependencies @@ -407,11 +517,10 @@ def _readDependencies(initFile): """ Reads in the library list using config parsing. @ In, None - @ Out, configparser.ConfigParser, configurations read in + @ Out, xml.etree.ElementTree.Element, configurations read in the most basic form """ - config = configparser.ConfigParser(allow_no_value=True) - config.read(initFile) - return config + root = ET.parse(initFile).getroot() + return root if __name__ == '__main__': mainParser = argparse.ArgumentParser(description='RAVEN Library Handler') From 8bf8ffc1c7576ad8f0de2e6136f5125057f45b2d Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 20 Jul 2020 12:04:03 -0600 Subject: [PATCH 11/18] moved to XML library dependency lists, see raven example for instructions --- scripts/library_handler.py | 247 +++++++++++++++---------------------- 1 file changed, 102 insertions(+), 145 deletions(-) diff --git a/scripts/library_handler.py b/scripts/library_handler.py index 2f9f7ec6e1..33f0b4f58e 100644 --- a/scripts/library_handler.py +++ b/scripts/library_handler.py @@ -90,16 +90,16 @@ def checkLibraries(buildReport=False): @ Out, missing, list(tuple(str, str)), list of missing libraries and needed versions @ Out, notQA, list(tuple(str, str, str)), mismatched versions as (libs, need version, found version) """ - missing = [] - notQA = [] + missing = [] # libraries that are not present, that should be + notQA = [] # libraries that are not the correct version, but are present plugins = pluginHandler.getInstalledPlugins() need = getRequiredLibs(plugins=plugins) - skipCheckLibs = getSkipCheckLibs(plugins=plugins) messages = [] - for lib, needVersion in need.items(): + for lib, request in need.items(): # some libs aren't checked from within python - if lib in skipCheckLibs: + if request['skip_check']: continue + needVersion = request['version'] found, msg, foundVersion = checkSingleLibrary(lib, version=needVersion) if not found: missing.append((lib, needVersion)) @@ -210,18 +210,19 @@ def findLibAndVersionSubprocess(lib, version=None): foundVersion = None return True, foundExists, foundVersion -def getRequiredLibs(useOS=None, installMethod=None, addOptional=False, limit=None, plugins=None): +def getRequiredLibs(useOS=None, installMethod=None, addOptional=False, limitSources=None, plugins=None): """ Assembles dictionary of required libraries. @ In, useOS, str, optional, if provided then assume given operating system @ In, installMethod, str, optional, if provided then assume given install method @ In, addOptional, bool, optional, if True then add optional libraries to list - @ In, limit, list(str), optional, limit sections that are read in + @ In, limitSources, list(str), optional, limit sections that are read in @ In, plugins, list(tuple(str,str)), optional, plugins (name, location) that should be added to the required libs @ Out, libs, dict, dictionary of libraries {name: version} """ - # OLD config = _readDependencies(mainConfigFile) + if plugins is None: + plugins = [] opSys = _getOperatingSystem(override=useOS) install = _getInstallMethod(override=installMethod) mainConfigFile = os.path.abspath(os.path.expanduser(os.path.join(os.path.dirname(__file__), @@ -231,19 +232,7 @@ def getRequiredLibs(useOS=None, installMethod=None, addOptional=False, limit=Non pluginConfigFile = os.path.join(pluginLoc, 'dependencies.xml') if os.path.isfile(pluginConfigFile): sourceFiles.append(pluginConfigFile) - config = _combineSources(sourceFiles, opSys, install, addOptional=addOptional, limit=limit) - libs = _parseLibs(config) #, opSys, install, addOptional=addOptional, limit=limit) - - #### OLD #### - libs = _parseLibs(config, opSys, install, addOptional=addOptional, limit=limit) - # extend config with plugin libs - for pluginName, pluginLoc in plugins: - pluginConfigFile = os.path.join(pluginLoc, 'dependencies.xml') - if os.path.isfile(pluginConfigFile): - pluginConfig = _readDependencies(pluginConfigFile) - pluginLibs = _parseLibs(pluginConfig, opSys, install, addOptional=addOptional, limit=limit) - pluginLibs = _checkForUpdates(libs, pluginLibs, pluginName) - libs.update(pluginLibs) + libs = _combineSources(sourceFiles, opSys, install, addOptional=addOptional, limitSources=limitSources) return libs def getSkipCheckLibs(plugins=None): @@ -354,19 +343,18 @@ def _getInstallMethod(override=None): # no suggestion given, so we assume conda return 'conda' -def _combineSources(sources, opSys, install, addOptional=False, limit=None): +def _combineSources(sources, opSys, install, addOptional=False, limitSources=None): """ Parses config file to get libraries to install, using given options. @ In, sources, list(str), full-path dependency file locations @ In, opSys, str, operating system (not checked) @ In, install, str, installation method (not checked) @ In, addOptional, bool, optional, if True then include optional libraries - @ In, limit, list(str), optional, if provided then only read the given sections - @ In, plugins, list(tuple(str,configParser.configParser)), optional, plugins (name, config) that should be added to the parsing + @ In, limitSources, list(str), optional, limit sections that are read in @ Out, config, dict, dictionary of libraries {name: version} """ - config = dict() + config = {} toRemove = [] for source in sources: src = _readDependencies(source) @@ -374,126 +362,92 @@ def _combineSources(sources, opSys, install, addOptional=False, limit=None): root = src.find('main') if root is not None: for libNode in root: - tag = libNode.tag - # check OS - ## note that None means "mac,os,linux" in this case - libOS = libNode.attrib.get('os', None) - # does library have a specified OS? - if libOS is not None: - # if this library's OS's don't match the requested OS, then we move on - if opSys not in [x.lower().strip() for x in libOS.split(',')]: - continue - # check optional - ## note that None means "not optional" in this case - ## further note anything besides "True" is taken to mean "not optional" - libOptional = libNode.attrib.get('optional', None) - if libOptional.strip().lower() == 'true' and not addOptional: - continue - # otherwise, we have a valid request to handle - text = root.text - if text is not None: - text = text.strip().lower() - # check for removal - ## this says the library should be removed from the existing list, which we do at the end! - if text == 'remove': - toRemove.append(tag) - continue - else: - version = text - libSource = libNode.attrib.get('source', None) - libSkipCheck = libNode.attrib.get('skip_check', None) - request = {'source': libSource, 'skip_check': libSkipCheck, 'version': version} - # does this entry already exist? - if tag in config: - existing = config[tag] - # check if either existing or requested is default (None) for each of the dictionary entries - for requestEntry, requestValue in request.items(): - existingFlexible = existing['version'] in [None, ''] - requestFlexible = libVersion['version'] in [None, ''] - # if niether is flexible AND they conflict, we need to error! - if not existingFlexible and not requestFlexible and existing != libVersion: - raise IOError(f'Dependency "{tag}" has conflicting requirements ({existing} vs {text})! Note the conflict may come from a plugin.') - # if new request is flexible, take the pinned one - if requestFlexible: - config[tag] = existing - # otherwise, take the new request. - else: - config[tag] = text - # if doesn't already exist, just add it to the queue - else: - config[tag] = existing - - TODO - else: - # if using an alternate install (not main), search through alternates - for candidate in srs.findall(alternate): - if alternate.attrib['name'] == install: - root = candidate - break - else: - root = None - - -def _parseLibs(node, existing, opSys, install, addOptional=False, limit=None, plugins=None): - """ - Parses lib file to get libraries to install, using given options. - @ In, config, xml.etree.ElementTree.Element, read-in dependencies - @ In, opSys, str, operating system (not checked) - @ In, install, str, installation method (not checked) - @ In, addOptional, bool, optional, if True then include optional libraries - @ In, limit, list(str), optional, if provided then only read the given sections - @ In, plugins, list(tuple(str,configParser.configParser)), optional, plugins (name, config) - that should be added to the parsing - @ Out, libs, dict, dictionary of libraries {name: version} - """ - libs = OrderedDict() - # find all the applicable libs for this opSys, install method, optionals - if install == 'main': - root = config.find('main') - else: - for candidate in config.findall('alternate'): + _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources) + # if using alternate install, load modifications + ## find matching install node, if any + for candidate in src.findall('alternate'): if candidate.attrib['name'] == install: - root = candidate + altRoot = candidate break else: - return None - - -def OLD_____parseLibs(config, opSys, install, addOptional=False, limit=None, plugins=None): - """ - Parses config file to get libraries to install, using given options. - @ In, config, configparser.ConfigParser, read-in dependencies + altRoot = None + if altRoot is not None: + for libNode in altRoot: + _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources) + # remove stuff in toRemove + for entry in toRemove: + config.pop(entry, None) + return config + +def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources): + """ + Reads a single library request node into existing config + @ In, libNode, xml.etree.ElementTree.Element, node with library request + @ In, config, dict, mapping of existing configuration requests + @ In, toRemove, list, list of library names to be remeoved at the end @ In, opSys, str, operating system (not checked) @ In, install, str, installation method (not checked) @ In, addOptional, bool, optional, if True then include optional libraries - @ In, limit, list(str), optional, if provided then only read the given sections - @ In, plugins, list(tuple(str,configParser.configParser)), optional, plugins (name, config) - that should be added to the parsing - @ Out, libs, dict, dictionary of libraries {name: version} - """ - libs = OrderedDict() - # get the main libraries, depending on request - for src in ['core', 'forge', 'pip']: - if config.has_section(src) and (True if limit is None else (src in limit)): - _addLibsFromSection(config.items(src), libs) - # os-specific are part of 'core' right now (if not explicitly reported in the pip section) - if config.has_section(opSys) and (True if limit is None else ('core' in limit)): - _addLibsFromSection(config.items(opSys), libs) - # os-specific of specific installer (e.g. pip) - if limit: - for lim in limit: - instSpecOp = "{opSys}-{lim}".format(lim=lim, opSys=opSys) - if config.has_section(instSpecOp): - _addLibsFromSection(config.items(instSpecOp), libs) - # optional are part of 'core' right now, but leave that up to the requester? - if addOptional and config.has_section('optional'): - _addLibsFromSection(config.items('optional'), libs) - if install == 'pip' and config.has_section('pip-install'): - _addLibsFromSection(config.items('pip-install'), libs) - instSpecOp = "{opSys}-pip".format(opSys=opSys) - if config.has_section(instSpecOp): - _addLibsFromSection(config.items(instSpecOp), libs) - return libs + @ In, limitSources, list(str), limit sources to those in this list (or defaults if None) + @ Out, None + """ + tag = libNode.tag + # FIXME check if library already in the toRemove pile; if so, don't check it? + # check OS + ## note that None means "mac,os,linux" in this case + libOS = libNode.attrib.get('os', None) + # does library have a specified OS? + if libOS is not None: + # if this library's OS's don't match the requested OS, then we move on + if opSys not in [x.lower().strip() for x in libOS.split(',')]: + return # nothing to do + # check optional + ## note that None means "not optional" in this case + ## further note anything besides "True" is taken to mean "not optional" + libOptional = libNode.attrib.get('optional', None) + if libOptional is not None and libOptional.strip().lower() == 'true' and not addOptional: + return # nothing to do + # check limited sources + libSource = libNode.attrib.get('source', None) + if libSource is None: + libSource = 'conda' # DEFAULT + if limitSources is not None and libSource not in limitSources: + return # nothing to do + # otherwise, we have a valid request to handle + text = libNode.text + if text is not None: + text = text.strip().lower() + # check for removal + ## this says the library should be removed from the existing list, which we do at the end! + if text == 'remove': + toRemove.append(tag) + return + libVersion = text + libSkipCheck = libNode.attrib.get('skip_check', None) + request = {'skip_check': libSkipCheck, 'version': libVersion} + # does this entry already exist? + if tag in config: + existing = config[tag] + okay = True # tracks if duplicate entry is okay or error needs raising + # check if either existing or requested is default (None) for each of the dictionary entries + for entry, requestValue in request.items(): + existValue = existing[entry] + # duplicates might be okay; for example, if the request/existing are identical + if requestValue != existValue: + # also okay if one of them is None (defaulting) + if None in [requestValue, existValue]: + # at least one is defaulting, so use the non-default one + if requestValue is None: + request[entry] = existValue + # if existValue is None, then keep the requestValue + else: + # there is a request conflict + print(f'ERROR: Dependency "{tag}" has conflicting requirements for "{entry}" ({existValue} vs {requestValue})! Note the conflict may come from a plugin.') + okay = False + if not okay: + raise KeyError('There were errors resolving library handling requests; see above.') + # END if tag in config + config[tag] = request def _addLibsFromSection(configSection, libs): """ @@ -591,7 +545,8 @@ def _readDependencies(initFile): libs = getRequiredLibs(useOS=args.useOS, installMethod='conda', addOptional=args.addOptional, plugins=plugins) msg = '\\begin{itemize}\n' - for lib, version in libs.items(): + for lib, request in libs.items(): + version = request['version'] msg += ' \\item {}{}\n'.format( lib.replace('_', '\\_'), ('' if version is None else '-'+version)) msg += '\\end{itemize}' @@ -608,7 +563,7 @@ def _readDependencies(initFile): # from defaults src = '-c defaults' addOptional = args.addOptional - limit = ['core'] + limit = ['conda'] elif args.subset == 'forge': # take libs from conda-forge src = '-c conda-forge ' @@ -623,7 +578,7 @@ def _readDependencies(initFile): libs = getRequiredLibs(useOS=args.useOS, installMethod='conda', addOptional=addOptional, - limit=limit, + limitSources=limit, plugins=plugins) # conda can create, install, or list if args.action == 'create': @@ -631,6 +586,7 @@ def _readDependencies(initFile): elif args.action == 'install': action = 'install' elif args.action == 'list': + action = 'list' preamble = '' actionArgs = actionArgs.format(env=envName, src=src) elif args.installer == 'pip': @@ -640,18 +596,19 @@ def _readDependencies(initFile): libs = getRequiredLibs(useOS=args.useOS, installMethod='pip', addOptional=args.addOptional, - limit=None, + limitSources=None, plugins=plugins) if args.action == 'install': action = 'install' elif args.action == 'list': + action = 'list' preamble = '' preamble = preamble.format(installer=installer, action=action, args=actionArgs) libTexts = ' '.join(['{lib}{ver}' .format(lib=lib, - ver=('{}{}'.format(equals, ver) if ver is not None else '')) - for lib, ver in libs.items()]) + ver=(f'{equals}{request["version"]}' if request['version'] is not None else '')) + for lib, request in libs.items()]) if len(libTexts) > 0: print(preamble + libTexts) else: From 02e6c265f9c917489b9a2e468792063f8669d56f Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 20 Jul 2020 12:22:09 -0600 Subject: [PATCH 12/18] attempting to find problem with printing on testing machines --- scripts/library_handler.py | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/scripts/library_handler.py b/scripts/library_handler.py index 33f0b4f58e..b53bf99c8e 100644 --- a/scripts/library_handler.py +++ b/scripts/library_handler.py @@ -357,12 +357,13 @@ def _combineSources(sources, opSys, install, addOptional=False, limitSources=Non config = {} toRemove = [] for source in sources: + requestor = os.path.basename(os.path.dirname(source)) src = _readDependencies(source) # always load main, if present root = src.find('main') if root is not None: for libNode in root: - _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources) + _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources, requestor) # if using alternate install, load modifications ## find matching install node, if any for candidate in src.findall('alternate'): @@ -373,13 +374,13 @@ def _combineSources(sources, opSys, install, addOptional=False, limitSources=Non altRoot = None if altRoot is not None: for libNode in altRoot: - _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources) + _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources, requestor) # remove stuff in toRemove for entry in toRemove: config.pop(entry, None) return config -def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources): +def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources, requestor): """ Reads a single library request node into existing config @ In, libNode, xml.etree.ElementTree.Element, node with library request @@ -389,6 +390,7 @@ def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources): @ In, install, str, installation method (not checked) @ In, addOptional, bool, optional, if True then include optional libraries @ In, limitSources, list(str), limit sources to those in this list (or defaults if None) + @ In, requestor, str, name of requesting plugin (for error verbosity) @ Out, None """ tag = libNode.tag @@ -424,13 +426,15 @@ def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources): return libVersion = text libSkipCheck = libNode.attrib.get('skip_check', None) - request = {'skip_check': libSkipCheck, 'version': libVersion} + request = {'skip_check': libSkipCheck, 'version': libVersion, 'requestor': requestor} # does this entry already exist? if tag in config: existing = config[tag] okay = True # tracks if duplicate entry is okay or error needs raising # check if either existing or requested is default (None) for each of the dictionary entries for entry, requestValue in request.items(): + if entry in ['requestor']: + continue existValue = existing[entry] # duplicates might be okay; for example, if the request/existing are identical if requestValue != existValue: @@ -442,31 +446,14 @@ def _readLibNode(libNode, config, toRemove, opSys, addOptional, limitSources): # if existValue is None, then keep the requestValue else: # there is a request conflict - print(f'ERROR: Dependency "{tag}" has conflicting requirements for "{entry}" ({existValue} vs {requestValue})! Note the conflict may come from a plugin.') + print('ERROR: Dependency "{t}" has conflicting requirements for "{e}"'.format(t=tag, e=entry), + '({ev} in "{es}" vs {rv} in "{rs}")!'.format(ev=existValue, es=existing['requestor'], rv=requestValue, rs=requestor)) okay = False if not okay: raise KeyError('There were errors resolving library handling requests; see above.') # END if tag in config config[tag] = request -def _addLibsFromSection(configSection, libs): - """ - Reads in libraries for a section of the config. - @ In, configSection, dict, libs: versions - @ In, libs, dict, libraries tracking dict - @ Out, None (changes libs in place) - """ - for lib, version in configSection: - #if lib not in configSection: - # return - if version == 'remove': - libs.pop(lib, None) - else: - # python 3 fix to work with python 2 syntax - if version is not None and version.strip() == '': - version = None - libs[lib] = version - def _readDependencies(initFile): """ Reads in the library list using config parsing. From bc4151de55b945601a9107bcb66bf1f5a80c24e9 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 20 Jul 2020 12:25:20 -0600 Subject: [PATCH 13/18] and again, looks like we have some old pythons on the test machines --- scripts/library_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/library_handler.py b/scripts/library_handler.py index b53bf99c8e..736f6905d8 100644 --- a/scripts/library_handler.py +++ b/scripts/library_handler.py @@ -594,7 +594,7 @@ def _readDependencies(initFile): preamble = preamble.format(installer=installer, action=action, args=actionArgs) libTexts = ' '.join(['{lib}{ver}' .format(lib=lib, - ver=(f'{equals}{request["version"]}' if request['version'] is not None else '')) + ver=('{e}{r}'.format(e=equals, r=request['version']) if request['version'] is not None else '')) for lib, request in libs.items()]) if len(libTexts) > 0: print(preamble + libTexts) From ba1bdd270b0207939a8c840f00cd3d0c7cc5ff81 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 20 Jul 2020 12:28:20 -0600 Subject: [PATCH 14/18] corrected lib version, also checked others carefully --- dependencies.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dependencies.xml b/dependencies.xml index 0f6da82002..ee61295ab1 100644 --- a/dependencies.xml +++ b/dependencies.xml @@ -39,7 +39,7 @@ Note all install methods after "main" take 1.4.2 3.1.1 0.9.0 - 1.9.1 + 1.1.1 1.13.1 3 From 91911fab4e28a47a6a588f43840c97bb5f1e4f15 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Mon, 20 Jul 2020 13:07:10 -0600 Subject: [PATCH 15/18] new heron update using xml --- plugins/heron | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/heron b/plugins/heron index 1c78c5a823..cee70f43f4 160000 --- a/plugins/heron +++ b/plugins/heron @@ -1 +1 @@ -Subproject commit 1c78c5a8230661e48ec84c1edc43aabc065af902 +Subproject commit cee70f43f4884f0bbb8f9c0c9bb61ca9ae2bdb21 From 1603459f44309a8372107da5408b5ffb0d03d9e0 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Tue, 21 Jul 2020 10:21:02 -0600 Subject: [PATCH 16/18] changed makefile ref from ini to xml --- doc/user_manual/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/user_manual/Makefile b/doc/user_manual/Makefile index bdde95c03d..8bb0b00774 100644 --- a/doc/user_manual/Makefile +++ b/doc/user_manual/Makefile @@ -6,7 +6,7 @@ introduction.tex raven_user_manual.tex model.tex runInfo.tex libraries.tex DataM Installation/clone.tex Installation/conda.tex Installation/linux.tex Installation/macosx.tex Installation/main.tex \ Installation/overview.tex Installation/windows.tex advanced_users_templates.tex LATEX_FLAGS=-interaction=nonstopmode -LIB_FILES = ../../scripts/library_handler.py ../../dependencies.ini +LIB_FILES = ../../scripts/library_handler.py ../../dependencies.xml all: raven_user_manual.pdf @@ -43,4 +43,4 @@ conda_command.txt : $(LIB_FILES) ./create_command.sh pip_commands.txt : $(LIB_FILES) - ./create_pip_commands.sh \ No newline at end of file + ./create_pip_commands.sh From 8afc5d4e5354b00995014ce7241cac682d7a0222 Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Tue, 21 Jul 2020 12:23:26 -0600 Subject: [PATCH 17/18] conversion script --- dependencies.ini | 100 ++++++ scripts/conversionScripts/deps_ini_to_xml.py | 307 +++++++++++++++++++ 2 files changed, 407 insertions(+) create mode 100644 dependencies.ini create mode 100644 scripts/conversionScripts/deps_ini_to_xml.py diff --git a/dependencies.ini b/dependencies.ini new file mode 100644 index 0000000000..b3c3f3c1e7 --- /dev/null +++ b/dependencies.ini @@ -0,0 +1,100 @@ +# Instructions: +# +# add library: library = pinned.version.number +# add with no pinned version: library +# remove a library: library = remove +# +# end instructions + +# Reading flow: +# +# The library handler will proceed in the following pattern for reading this file: +# 1. Read in "core" (stuff that comes from conda) +# 2. Read in "forge" (stuff that comes from conda-forge) +# 3. Read in "pip" (stuff that comes from pip) (NOTE: not the pip-only install!) +# 4. If optional, then read in "optional" +# 5. If OS is given, read in either "windows", "mac", or "linux" +# 6. If pip-only install, then read in "pip-install" +# +# Note that each reading-in stage might involve adding or removing libraries, in order. +# If a library is not present but requested to remove, that's fine, no crash, move on. +# +# For more information about the command line arguments to library_handler.sh, see that module. +# Also try +# `python library_handler.sh -h` and +# `python library_handler.sh conda -h` +# +# end reading flow + +##### Typical installation (conda) +# main libraries from main conda +[core] +h5py = 2.9.0 +numpy = 1.18.1 +scipy = 1.2.1 +scikit-learn = 0.21.2 +pandas = 0.24.2 +xarray = 0.12.1 +netcdf4 = 1.4.2 +matplotlib = 3.1.1 +statsmodels = 0.9.0 +cloudpickle = 1.1.1 +tensorflow = 1.13.1 +python = 3 +hdf5 +swig +pylint +coverage +lxml +psutil +pip +importlib_metadata + +# secondary installs from conda forge +[forge] +pyside2 + +# tertiary installs directly from pip +[pip] + +# optional libraries +[optional] +pillow = 6.0.0 + +# operating system alterations (conda) +[windows] +# nothing currently + +[mac] +# nothing currently + +[linux] +# these prevent Intel crash errors with mkl +nomkl +numexpr + +# operating system alterations (pip) +[windows-pip] +# nothing currently + +[mac-pip] +ray + +[linux-pip] +ray + +##### Alternate installs +# install using pip +[pip-install] +hdf5 = remove +swig = remove +pip = remove +python = remove +nomkl = remove +numexpr = remove + +[skip-check] +python +hdf5 +swig +nomkl diff --git a/scripts/conversionScripts/deps_ini_to_xml.py b/scripts/conversionScripts/deps_ini_to_xml.py new file mode 100644 index 0000000000..153d085d92 --- /dev/null +++ b/scripts/conversionScripts/deps_ini_to_xml.py @@ -0,0 +1,307 @@ + +""" + Converts old dependency files from .ini to .xml +""" + +import os +import sys +import configparser +import xml.etree.ElementTree as ET + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), '..', '..', 'framework'))) +from utils import xmlUtils + +def readConfig(inFile): + """ + Reads in ini file. + @ In, inFile, str, path to file object + @ Out, config, ConfigParser, config read in + """ + config = configparser.ConfigParser(allow_no_value=True) + config.read(inFile) + return config + +def convert(config): + """ + Converts config to xml + @ In, config, ConfigParser, old dependencies + @ Out, xml, ET.Element, converted xml + """ + xml = ET.Element('dependencies') + + addCore(config, xml) + addForge(config, xml) + addPip(config, xml) + addOptional(config, xml) + + addWindows(config, xml) + addMac(config, xml) + addLinux(config, xml) + + addWindowsPip(config, xml) + addMacPip(config, xml) + addLinuxPip(config, xml) + + addPipInstall(config,xml) + addSkipCheck(config,xml) + + #print('DEBUGG new xml:') + #print(xmlUtils.prettify(xml)) + return xml + +def addCore(config, xml): + """ + Adds core entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'core' in config: + node = ET.Element('main') + xml.append(node) + print('DEBUGG entries:', list(config['core'].items())) + for lib, vrs in config['core'].items(): + new = ET.Element(lib) + node.append(new) + new.text = vrs + +def addForge(config, xml): + """ + Adds forge entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'forge' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['forge'].items(): + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['source'] = 'forge' + +def addPip(config, xml): + """ + Adds pip entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'pip' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['pip'].items(): + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['source'] = 'pip' + +def addOptional(config, xml): + """ + Adds optional entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'optional' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['optional'].items(): + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['optional'] = 'True' + +def addWindows(config, xml): + """ + Adds windows entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'windows' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['windows'].items(): + new = main.find(lib) + if new is None: + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['os'] = 'windows' + else: + new.attrib['os'] = new.attrib['os']+',windows' + +def addMac(config, xml): + """ + Adds mac entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'mac' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['mac'].items(): + new = main.find(lib) + if new is None: + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['os'] = 'mac' + else: + new.attrib['os'] = new.attrib['os']+',mac' + +def addLinux(config, xml): + """ + Adds linux entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'linux' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['linux'].items(): + new = main.find(lib) + if new is None: + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['os'] = 'linux' + else: + new.attrib['os'] = new.attrib['os']+',linux' + +def addWindowsPip(config, xml): + """ + Adds windows pip entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'windows-pip' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['windows-pip'].items(): + new = main.find(lib) + if new is None: + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['source'] = 'pip' + new.attrib['os'] = 'windows' + else: + new.attrib['os'] = new.attrib['os']+',windows' + +def addMacPip(config, xml): + """ + Adds mac pip entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'mac-pip' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['mac-pip'].items(): + new = main.find(lib) + if new is None: + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['source'] = 'pip' + new.attrib['os'] = 'mac' + else: + new.attrib['os'] = new.attrib['os']+',mac' + +def addLinuxPip(config, xml): + """ + Adds linux pip entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'linux-pip' in config: + main = xml.find('main') + if main is None: + xml.append(ET.Element('main')) + for lib, vrs in config['linux-pip'].items(): + new = main.find(lib) + if new is None: + new = ET.Element(lib) + main.append(new) + new.text = vrs + new.attrib['source'] = 'pip' + new.attrib['os'] = 'linux' + else: + new.attrib['os'] = new.attrib['os']+',linux' + +def addPipInstall(config, xml): + """ + Adds pip alternate install entries to xml + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'pip-install' in config: + alt = ET.Element('alternate') + xml.append(alt) + alt.attrib['name'] = 'pip' + for lib, vrs in config['pip-install'].items(): + new = ET.Element(lib) + alt.append(new) + new.text = vrs + +def addSkipCheck(config, xml): + """ + Adds skip check flags + @ In, config, ConfigParser, original config + @ In, xml, ET.Element, root of dependencies + @ Out, None + """ + if 'skip-check' in config: + for lib, vrs in config['skip-check'].items(): + for install in xml: + for exists in install: + if exists.tag == lib: + exists.attrib['skip_check'] = 'True' + break + +def write(xml, origPath): + """ + Write new dep file + @ In, xml, ET.Element, filled XML with dependencies + @ In, origPath, str, original file path + @ Out, None + """ + newPath = os.path.join(os.path.dirname(origPath), 'dependencies.xml') + xmlUtils.toFile(newPath, xml) + print('Wrote new "dependencies.xml" to {}'.format(newPath)) + + + +if __name__ == '__main__': + usageMessage = 'converter usage: python deps_ini_to_xml.py /path/to/dependencies.ini' + if len(sys.argv) != 2: + print(usageMessage) + sys.exit(1) + oldPath = os.path.abspath(sys.argv[1]) + if not os.path.isfile(oldPath): + print(usageMessage) + print('ERROR: File not found:', oldPath) + sys.exit(1) + config = readConfig(oldPath) + xml = convert(config) + write(xml, oldPath) + + + From bc6f2820988626057cc87e1d5a43c0c63ead8fea Mon Sep 17 00:00:00 2001 From: "Paul W. Talbot" Date: Tue, 21 Jul 2020 12:23:59 -0600 Subject: [PATCH 18/18] removed old file --- dependencies.ini | 100 ----------------------------------------------- 1 file changed, 100 deletions(-) delete mode 100644 dependencies.ini diff --git a/dependencies.ini b/dependencies.ini deleted file mode 100644 index b3c3f3c1e7..0000000000 --- a/dependencies.ini +++ /dev/null @@ -1,100 +0,0 @@ -# Instructions: -# -# add library: library = pinned.version.number -# add with no pinned version: library -# remove a library: library = remove -# -# end instructions - -# Reading flow: -# -# The library handler will proceed in the following pattern for reading this file: -# 1. Read in "core" (stuff that comes from conda) -# 2. Read in "forge" (stuff that comes from conda-forge) -# 3. Read in "pip" (stuff that comes from pip) (NOTE: not the pip-only install!) -# 4. If optional, then read in "optional" -# 5. If OS is given, read in either "windows", "mac", or "linux" -# 6. If pip-only install, then read in "pip-install" -# -# Note that each reading-in stage might involve adding or removing libraries, in order. -# If a library is not present but requested to remove, that's fine, no crash, move on. -# -# For more information about the command line arguments to library_handler.sh, see that module. -# Also try -# `python library_handler.sh -h` and -# `python library_handler.sh conda -h` -# -# end reading flow - -##### Typical installation (conda) -# main libraries from main conda -[core] -h5py = 2.9.0 -numpy = 1.18.1 -scipy = 1.2.1 -scikit-learn = 0.21.2 -pandas = 0.24.2 -xarray = 0.12.1 -netcdf4 = 1.4.2 -matplotlib = 3.1.1 -statsmodels = 0.9.0 -cloudpickle = 1.1.1 -tensorflow = 1.13.1 -python = 3 -hdf5 -swig -pylint -coverage -lxml -psutil -pip -importlib_metadata - -# secondary installs from conda forge -[forge] -pyside2 - -# tertiary installs directly from pip -[pip] - -# optional libraries -[optional] -pillow = 6.0.0 - -# operating system alterations (conda) -[windows] -# nothing currently - -[mac] -# nothing currently - -[linux] -# these prevent Intel crash errors with mkl -nomkl -numexpr - -# operating system alterations (pip) -[windows-pip] -# nothing currently - -[mac-pip] -ray - -[linux-pip] -ray - -##### Alternate installs -# install using pip -[pip-install] -hdf5 = remove -swig = remove -pip = remove -python = remove -nomkl = remove -numexpr = remove - -[skip-check] -python -hdf5 -swig -nomkl