From 2711e61cb8515c0d19439bc35c72c2fc1ad73ccb Mon Sep 17 00:00:00 2001 From: Quentin Codelupi <quentin.codelupi@cern.ch> Date: Tue, 23 Jul 2019 15:41:38 +0200 Subject: [PATCH] Init --- .idea/other.xml | 7 +++++++ nxcals-hadoop-pro-config-0.1.164.jar | Bin 94 -> 4381 bytes nxcals-jars | 1 - property-pull.py | 20 +++++++++----------- 4 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 .idea/other.xml mode change 120000 => 100644 nxcals-hadoop-pro-config-0.1.164.jar delete mode 120000 nxcals-jars diff --git a/.idea/other.xml b/.idea/other.xml new file mode 100644 index 0000000..640fd80 --- /dev/null +++ b/.idea/other.xml @@ -0,0 +1,7 @@ +<?xml version="1.0" encoding="UTF-8"?> +<project version="4"> + <component name="PySciProjectComponent"> + <option name="PY_SCI_VIEW" value="true" /> + <option name="PY_SCI_VIEW_SUGGESTED" value="true" /> + </component> +</project> \ No newline at end of file diff --git a/nxcals-hadoop-pro-config-0.1.164.jar b/nxcals-hadoop-pro-config-0.1.164.jar deleted file mode 120000 index 7088c21..0000000 --- a/nxcals-hadoop-pro-config-0.1.164.jar +++ /dev/null @@ -1 +0,0 @@ -/home/dredd/Documents/CERN/spark-2.4.0-bin-hadoop2.7/jars/nxcals-hadoop-pro-config-0.1.164.jar \ No newline at end of file diff --git a/nxcals-hadoop-pro-config-0.1.164.jar b/nxcals-hadoop-pro-config-0.1.164.jar new file mode 100644 index 0000000000000000000000000000000000000000..8c0f0935004ac9703afb2b09a3c0f5e1539a9695 GIT binary patch literal 4381 zcmZ`-cQjmG_ny(E86{?d!7yrcBDyG}mtctAM-Zczh!~w9(R+&^N^}N65G6`PMvdNk zf3-BkZ}NR><^5u<-@fad`_F#%I(OfF&a<C$U!9QX3IISx2H;5B&;sC%;P-?#a{SOl zAmw;fwUqb?0r&rMtO+34qsKek!VjANb<~v8QdL6eA^9|wjx;~@YpC<`4P95~<^KE$ zSF0;9D!j0|v`TO(SX)3Np%@Uq2qFOBIzE_<i#v+f!@(29_ZaO|p4hI{DR3=(-!E#I zRX4jG3wu>T4erY3=rxW|EYL(r>NOW}OvizW6N{N67%)Ba`-a|)^u{ltaT8(Ane**J z^ZL<;1++Z*%Jt6@`X-h|!3>JmcaA<$WuVO?KEG{HG1e8zBH;4T!6a(SYrj%Xs5Z9U zG_BC}FFCSE%66_6x^|ro+9K7Kvc+STN4m>uP%n5^jJ!KKRm(kwI42(8Vh?46Y^!J~ z4eIcCFKr#gvS-c3e^m*(*r4o%RvM8ZvVyeBDTkffy@Hd5+2bg_^{<{q>DfbAS>;Oo zI!K%>7s=0dWvk{nyD&G-3OQ?1i^@1F#2W$KzM=ji>_4g`mxwU6=9QwtTR#5R>wFwk zm^A|;>H^==DkPCVc0A2$)vhv;2E2+>2*>)qG|=-mHWSp9O*tm~`+ouMt7*T(r?-;% zzbUr2we$Fs;sIl{_Ocl0*XPRKgGRS0`Bd=DuUb3|9NC77e$7k|*waO6BNOYgo%<_g zryZ`KcSKNTCCosm<dIa4uauk6HCfz9S0->7p8udndQ|$?Q-2Yp!OLg_ffNZS$8g2f zi^q~LSX?`%C9QpBOYGA|sK(Xsuy>FWpzrRG>p<Bx@qTwpYd8OuzDn2!E>mr>D9hkz z#hDl3L88g|!dWz~P-SY&Eh10GrBKq&TrA`Dkh>OkZEy;msC786E{#=?4uL1PHKF4E z3U*#na{J}}f?16`PL;OR_3#SG%X97ZB94%ec8+MA?9=(?i~de4vMfI#Kq%Esw!I%1 z>cXEPgqf7S*}(d1dK@wC4DDr|#5n}pmQjge%tu;6v2IByU*%uJg;j+7LUomkrA=!b zdj(u}!Gl=Vz-Z{QTrq(C)l|HQ#Dm(DAz#~JrdN9W8dvzBejh0YSSD!mAo*iRq}7-T z2nL?n_V(jUD$s%WaQEldKyHcJDOs_kM99Rjuv0gOD&MIR4duf%pI7+Z2%%*lnHJyB zFp3N%2XYS*CBhT$5cl~Wh5Zy~jX$#=Q0p!vTNir{OjbiUUa+#Udl}V9r7vZo&_Nr{ z<cGxqjH#gJ4`@xH{sEh-v2@eO7xgF5ExS$XTcGI!ceWgfc+72<bLrc?IRy%F_l<mB z_|%%#eC6254xV%3yNT4jNAlhqYf4_+ov$hzZsa!{dq*LH`Q;V{T|U@SN6_Uew$aBQ zu~h=er8B43N(IkWT-|+VqhpaD)V&6mjGZ6WQh0JhPI?T6%P47;lb#j+<6AL$6oc@8 zVQ}(Jb1Lx(-LMDQmXY)=Lp?9YIPHtB)y1N`K*vC|in!#@clkONul4V_B!2ZecIX|1 zaTY(x!>+2$E|!XCl5W-sruMx0$mN(@MQ`;$=#gdpNijO7g=6;4$_dD}Xx_qbyEAgL z%X9H-<SvKO@4Pa|lsa5F8JonZflv7iQpy;Dt>5^VX*QQ;*l;6D1yjS`xJS`(?kG(p zzoSd5<3^rL`VE^Zm{Z7kuV(6dWQ|S~j$#L1cO@y<20l{$n4BeQ=#)J8vrJmGCn59v zW1%DI(?)vVyxgs9pO|z{?m)EPWQ!>Ut^*8eW#+OCb09thmOk>!8v2Lsd~N<hX?dDs zEFCf{<A1uo=3TJZuq6DRa>=b*QN_>@L`~d&WuiLY@+jtQTs=~2P`vus@<|HuHf^Vi ze1{Z=Q;_b6`i{^SCPYpg(bHMu!^U1nPB6dh25rWS2r5`dEg>|mN?V2qisEATEMZBq zvB}hsKml*t-**ODpo5t6HhPxFi>j~JB#A~;St{~i)9O}Ta^Jh^Mq%!lie76_G4CNs z&K8E(fOE+>F<Kf_RSp$`bi-*>6+d$e$?i5G)&1fgNcFl`E!P<6uMkfNS7{tthK9p? zB46{;oxQz~pu6z(I6V$@-t#fHgSw8rUltE@_@c6ak-|8s-XmVUO8cEqPUnTmfSd2Q z0BDot+@H(mbaStx@~lUGW%u55=&Uz$g>dtf@q&NE!<OWX&L3BEPF|GG@W+>4XPsp+ zeP)=#uWRteTYXkep}qTp>Wt#=W9}I(GL05bV82KK017;lpsifpQMP{$y4U)eu~Pz| zmZRG&qk4#)fAp*MEu#Z(N&{{`anSBnC9$UKNkJPP?vFbJu6EL(d;1u1elB)GHm5kr z6fu*ys6(ttIcw$5nbPl#WBL;6_dWoohSCF_^*gnev(?^c@mNikhfA3ak10{1-*d_e zOhg6PeZDU54%B@pU9^;j(<TpLWR;i4k^?DWRCKg(EBPcU#U7bo>-vCt_~g!b%n@pa z>9bvBbi32f8py9K`fFjx@roFZ#7R%V98IO~j|S{C?Jc$VESDGiE7b=h2Ck$ZTde0V zvTckyhMjNXDmzZAj9^7WrD-bf1tZ14Vk^%Nj+JsmagvQM<(BMA%EsP)ZA#p@#mp*H zSBTm5YS?)gXY+9OJN?}(oT9)6s0XMMc;sDi)31rQ{`*FEn2bPqqzH<ej&@)2OHa?7 zY&PLGgy;2So7)fYDrw3uoVrd_BLq|+3e2bDbs$jba9iV18_*T93xf#o33m}(ebo8L zOO3j+FNIkTje>ax)S8`_ELQ98kj7si=6<b7clh&vR~mVgd=)bglE$5_%m>$0;8wAw z<N~B^iBWmsO7lSljZVu<Awy52NS2lBRB2T2*B@8F+cGutP48$E%`*$&y4O8lN@U}X zEo7X3OowWjbB8Cf-_EF*d8QWq#wt~8vGhE9fp#zgp}7<RqcwGWkl=}GhD~W!?$w*j zl^-re7yW?#T|zh7*0cEVOnC+JU#9f6a(DhyKpza}U6#Z^n0<?Qg;VSssC_K8$<Uqw z_9h1uh1g%s;Z98JGU14Eis%Rkut`X=&{89?k4DBvK5WI=1S|`l??*qA^x(cpeSj_v znJ(Y=0sGq^)cD2qVXvo#`a|dHMa=AdUOnz7XUm?gm!iZO#;N7C-^v6p_zleq(JQbZ z<X@Z5nTh=~{;lK?u5RE$n#Gw6f=UR(u3tMM^HE7~{cf$qIpcE|Y5>c^Xs|qKfD3ig z+f+YL+-@!f{#X!SbU0XedTUPhd5^3G>GFEfctjh=LYI9R|2OkkrRDYU#mC1Rm=>&& zcxFc?v|+Jxq`zWh%FgT7>_wK8%#H-wxZL?-R{QHlIocpq^D8$zp$T>nrednC82?Ot z&4K<$-%Nw4d_ZB@<OI`6pJD^UrzMl3R@to)tV^RJJbqML+PH>}Eu`U|=^N?0SyTwK z{P6yr^nqICZ#HQvo|K#5__#t1m~*Sm>7KDfZoVwY{gYJoea>B#H;UZSZx}*(xJ_;s zKp^pHL`dt^uzTs94aw%BbqVT-UVtaUlHv2jqCPW%4H*^nShxd@2@c>7>yt@lH?^7; z<b@Xsrn@`B%28^iLOx>l2hXwaIeN&m{g8o_67$n08^VF{0!C-HPh)+SU&PCaoL;(` zQ%0AzS7?)b)hcp@BlAw4+_Wbp{6O%T?UQwEkqn}_Nb5!ld0TNdvp=WDMDGJ=>GNwj zd~_Y)9508o84QiI21}U;xgi7dOI7+H@pwM4oE8D9!b=VYV3A{XsFGOM388a!bydN# z^C(5hcRZaYY1+axJDQ9Q0Z6bLEil?g#pR(7a?LNs$Z9#KN>);GUw!F&u`LAssaiC< zXBiuu37+$8J$WMiEj2A%LvGA0;||tk(%rLGL6#qle)n_cY%)w+|2$`f0D^)+RrI;j z%_WT5j7}jMaQjrYu9Q^$X+Y0;f14~xY~n*HNJ_yR$S>Sil&vm3KM^EBM&6q~pv;zc zmDwp~%&z>s-C%%W79R;>rAO)?L6OGDXFRrqT*C!$b$t{2-rzx)M~7ItkaJ4k<I!2M zdz^(iqE}M-DK?jSjc-q#=r>1$6W(neXd@4J07Qp$a#n|+V^`C(x^v#Np`@R`ChskF z=~Wp&n*=Non@X$4pdJxhzj4?M>gy*nobdi+&fChvGuz?rx%Ubv;BvtD1@<%sfr|~c zTeyqLc9kZfKezW_{E}05BX-xIPLR@1a(XAl%TqU|!W)%Zxt3H0#zyB8jYtP}_N_E! z1{yCsq|rb#u2X54v8>Z*>=tBM2-I;UlNag=pMpD2a2iIPw$9S~p$=zm)7AAFv(#9P zefd_{g7p;ALp<5FJouNj?^+>=9W<Vp#dw5s=nE55OqjrdG$mV><5vCv(??$W>|tcb z$Y<Q~#ibS;DBdJ9gI76+Bmlq-JU7`}TY3B|b6WIuTt);yn8G%em2@#xO1;qG8zmVN z)Va?4y<Yq;>bgW9`iY#MOsIB`_62#$e?Ry<|83T+)#Kx~&9q`{V1{~ilt;pc@omIr zex#@wVH1f!{3cDAHKB4MqZhUE9r?Pbbefy8h6UY0k=kNAc~VzT-%rCMWkLW2nj-Lg z_yKvCE-=Hn<=G4uG1|&(znH{5;Sl(qt&igNP(5mjBf>8hTV1tQI9Dpw@7@6*wmRuS znXl~UbTD-+t{T17Agw^8%Zwkgn@)JB(qWmvg71hOG;`~prZ-aCGOfFjZWPJ*F!GLd zb9v~f=E1{7DrY#^;TKv`1L9-dOkzfza*#vnMqo{?J~3p0Zw~xH-$0{^LfHC?o2N%a zLC<NZv7Y)$(OeMFA%vE1Z|nK&NzWwBIpM;M`&;9f=NwWxy~N8=qQH2@cud>*6Q2tQ zkJ@v2(FX7~jKP$&sbMJqssB7s%oPX(Mv}_kMBRzY*n5q%1-OxJodZlhI4d28brdR@ zr~AIR;q%aItGsEGm&t+n`Gc$hubsDCt>SDbg)>QdPP^OX6E3SU(Jp&m9(_A#h*~R( z`#8$2MeWxeKO?}l!bmrDUh01&>SjdfWIbTMXZ5{W)8Z@!1H7*eB%mSu|5Y}=81M!H z?8^Oq|Iyr+VV5=cZ|C1u5X6O_|G@r7fnSbrS!n+U{+|fm`1vo{eHnULK>iDz#Mks+ za`H0ta#Qv%^gVtnTx!uSgD=zZzhFZA1;t<Fd>MLq!~F|2#osdgz5VX16XVwc0FdIJ NeEbL5{_Y+D{|Ar#gk}H$ literal 0 HcmV?d00001 diff --git a/nxcals-jars b/nxcals-jars deleted file mode 120000 index b9ec078..0000000 --- a/nxcals-jars +++ /dev/null @@ -1 +0,0 @@ -/home/dredd/Documents/CERN/spark-2.4.0-bin-hadoop2.7/nxcals-jars \ No newline at end of file diff --git a/property-pull.py b/property-pull.py index 704ba7b..9185411 100644 --- a/property-pull.py +++ b/property-pull.py @@ -40,11 +40,9 @@ sums_cols = udf(lambda arr: 0 if arr == [] else __builtins__.sum(arr), IntegerTy ''' #TODO - -# Check Why Columns is not working -# Check compression ? -# Reduce ( UDA ) -# Scala UDF ? (working with python's lambda for now ) -# Write Brut hito data correctly => ['element'] not OK +# Check compression ? Done => MB optimisation using numpy +# Reduce ( UDAF ? ) +# Scala UDF ? (working with python's lambda for now ) ''' @@ -68,12 +66,12 @@ def pull_histogram(_start_time, _end_time, _device): data = {} df = KeyValuesQuery.builder(spark).system("CMW") \ - .startTime(_start_time) \ - .endTime(_end_time) \ - .entity() \ - .keyValue("device", _device) \ - .keyValue("property", _property) \ - .buildDataset() + .startTime(_start_time) \ + .endTime(_end_time) \ + .entity() \ + .keyValue("device", _device) \ + .keyValue("property", _property) \ + .buildDataset() df.printSchema() -- GitLab