From 9fd46088d5f52c95a632b9f3d8ebf473416d4859 Mon Sep 17 00:00:00 2001 From: Sambo Chea Date: Tue, 30 Mar 2021 09:16:11 +0700 Subject: [PATCH] Add excel2json function and updated custom mapper from props --- .gitignore | 4 ++ Dockerfile | 16 ++++++++ Makefile | 11 +++++ README.md | 60 ++++++++++++++++++++++++++++ build.sh | 3 ++ data/mapper.json | 18 +++++++++ data/people.xlsx | Bin 0 -> 8544 bytes excel2json.js | 102 +++++++++++++++++++++++++++++++++++++++++++++++ index.js | 16 ++++++++ package.json | 15 +++++++ 10 files changed, 245 insertions(+) create mode 100644 Dockerfile create mode 100644 Makefile create mode 100644 README.md create mode 100644 build.sh create mode 100644 data/mapper.json create mode 100644 data/people.xlsx create mode 100644 excel2json.js create mode 100644 index.js create mode 100644 package.json diff --git a/.gitignore b/.gitignore index 6704566..f2b6599 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,7 @@ dist # TernJS port file .tern-port + +yarn.lock +package-lock.json +outputs/ \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..393342e --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM cubetiq/calpine-node:latest +LABEL maintainer="sombochea@cubetiqs.com" + +VOLUME [ "/app/data" ] + +ENV INPUT_FILE './data/people.xlsx' +ENV OUTPUT_PATH './data/outputs' +ENV MAPPER_FILE './data/mapper.json' + +WORKDIR /app + +COPY . /app + +RUN yarn + +CMD [ "node" , "index.js"] \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..32140e5 --- /dev/null +++ b/Makefile @@ -0,0 +1,11 @@ +DOCKER_IMAGE=cubetiq/node-excel2json + +build: + @echo "Building docker image..." + docker build . -t ${DOCKER_IMAGE} + +run: + @echo "Running container..." + docker run --rm -t ${DOCKER_IMAGE} + +.PHONY: build diff --git a/README.md b/README.md new file mode 100644 index 0000000..aff9367 --- /dev/null +++ b/README.md @@ -0,0 +1,60 @@ +# excel2json + +- Read file excel to json +- Custom mapping with custom columns and configs + +# Build + +```shell +bash build +``` + +OR + +```shell +make build run +``` + +# Example + +```shell +docker run -v /my/path:/app/data --rm -it cubetiq/node-excel2json +``` + +```shell +docker run -v /home/sombochea/excel2json:/app/data -e APP_NAME="EXCEL 2 JSON" -e MAPPER_FILE="./data/mapper.json" --rm -it cubetiq/node-excel2json +``` + +# Mapper Config + +```json +{ + "data": [ + { + "dataIndex": "Name", + "label": "Name" + }, + { + "dataIndex": "Age", + "label": "Age" + } + ], + "configs": { + "outputPath": "./data/outputs/exported", + "outputName": "my_exported_data", + "sheetName": "Sheet1", + "saveToOutput": true + } +} +``` + +# Environment + +```env +APP_NAME=custom app name +INPUT_FILE=./data/mydata.xlsx +OUTPUT_PATH=./data/outputs +MAPPER_FILE=./data/mapper.json +SHEET_NAME=Sheet1 +ENCODING=utf-8 +``` diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..7ece719 --- /dev/null +++ b/build.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +make build run \ No newline at end of file diff --git a/data/mapper.json b/data/mapper.json new file mode 100644 index 0000000..80ff8e5 --- /dev/null +++ b/data/mapper.json @@ -0,0 +1,18 @@ +{ + "data": [ + { + "dataIndex": "Name", + "label": "Name" + }, + { + "dataIndex": "Age", + "label": "Age" + } + ], + "configs": { + "outputPath": "./data/outputs/exported", + "outputName": "my_exported_data", + "sheetName": "Sheet1", + "saveToOutput": true + } +} diff --git a/data/people.xlsx b/data/people.xlsx new file mode 100644 index 0000000000000000000000000000000000000000..91626964331ee236df50bb7a4e8ad6db20968943 GIT binary patch literal 8544 zcmeHs1y@|j)^-CSxVu|$cY+0XCj@tQ*PxBNy95Xjf(3U8u8q4xAUFhKP{BIRF*_0FVLXgdx%NApn55X8-^O02WeP)ZWg; z)Xqi!wWourvo4c|tqp1ZGf0|T03`VR|F!?(9VknZSLkLz4ZfCICOgBXw9_OF%5do% zL#Iiqtn*=UKGxq0Px3^aDkh6}QDp~e}U*nc` zD3E>%&qn?;eIKgRC((}cmd^U);P>GYqFl=HIcoj5PNv(TvFGk5JH3YS0!i(ba!8AF zkuexFEMiLd+JJqV_^rAeqy_j zpUr*RZIkB{4`h??X768hXNMz^$$lAT&u!B;g_banCR$Jj&3{cPo8Ogfpzp!5&DtW#s5xB$<{fz=KHte5&urZ&#ZOn>zM)5rf}CH&jc z%M#=jyIBxJPNeQb2d-yU;!s6o-G!xE$X@&UOE06;MZbSRu+l+Ii252o5K7Xo&G&w2 zX@x&}e}L?2lcOvW9g~lu-n~2|_0G{1hK|Z9MZ&RcqX*4x=5ppLT~fxA#;q-uuC%c@ zPj+yPLSpJvtO|9ENgW>!qmVEJTOd6^t505Y)##=IVn#&us63>yfjf6UaV*nsE~RJ> z=~dWExx>kHtN~{u^MwlU0UNT*TYNQDiO3IFa5M?IZlVOJ$ty}x=RPX4FV%cidVrd>USlnri}0D04vDH=Kuf-cxOCp znBDE2tc~pLt^aUb1!}g6ODw35f|gHYXV<8FTe`Jj2@<8xV55-*$rE|X#nB~WL<3Qu ze0B;f)zgY1#LJ1kkB$jkZw@bbs$=&dBqHI=Jd4MOLVQz5{$<7r=*x(~E+p~ZB6&jO zwGB{DGjR07T1U2Rkhd@T<^sUKBt5rI#nnVpzS)e@RzVCz7q=yn^14%{qO2$-$_$jj zs)n5sU1gq}q;4S;w#AA|KIS=O6kktSIt?J#6vi62pKZ01sAUs)UQuw?R1SKb#3{so z3M`!?#>6wlXMB?!ckvO>z_*E?9rO;y4`^5^DNA~1QnK|CGkRbKPDmSSPw^4M2F(c- zc&9@?T~@)7tkYti3|0-IQ}I3?P#wMvw4s~pRb9g$j+NQg&hn= zMGjkh(VtxfKOyJE>m|W2mR!zKM!yt}QRtGK;5?Fm>r@`DX?x&q^^p_s!z>Xok@Kb-^W=vvc=ThMs2@)hV##{T1iNaQtsai02)`Ou?9Nu$U^%E|- zjVQs?fo$U*^Z^3Y0coXzE@#QR=r>NJ{LX=RAtfx?K`-JeP%~@Nchc;#vYG|MnRo2+ zrwUv2rY%SABYO!o9xNr)=<`ZNOfL_lVh7#a8`4n-mJA48f_KfA=e8a0^dsEIXAs;9 zoSgg$Hn%~SXLnL7rMG9NWD`TP`fYWjWTXUBM$!7|d-h-^|IYOQ!KBo4@Jut9@Av>% z2r$?G;}ibQ_`m!E1UOy;Ys`OlD^rn|?O{P}LwF2janE$aLR)ZQCO=d=K!F{srCp*T zWB0vSBcN;6)tHrKhO!UwJOTB)U2`IDLSbBX(icadKl8${IpKr02c3hUV2zH-iGh;P z80d%l2c<^{$QjNUO@avn3^|xQUvFqA@j~Ahk&9h4O@?q-jk371Svijr4T0_BaP5}= zU6ATwIk2{&fi#2GZ2PH6bm{8(JD?}Y#5o?EG(VCjU}ATgjQ*xm{0jE0_1;Y0bJV)? z{T<>qVW})}38#Q}hzn`xfFy|@ZG!W5etL`A$%c!__aWMjr89=%ixac0!Ty;w@Jatp znY{Fki&QXOguvzz7yQPb=F!>0)YQe9`NzQehn>ty7_{qRK@B-czC%tqSr00wB+re2 zPu89}I_AMJ+Muih6_uQV7jbdGCV?SRhgE?y9n!V2(q z2x2T#R;x+@6DjTM!#~$=HEc=q))!5(|i%{*V!8hLwlWz7PA}za7)*x;{k$&XUj_5ZK5*10Y z$M#st;u!lnET=6H5)Hsi#v(t!4Ho*;JMUEp} zVF)i%=yK|jSSZ6+_$!J8o3sZQ6r074iL$5X10oqPImW!ZHn;DrY%G(*Lu%1vLt38|92_PwU{L3x)wPQ)p8g^8Ba^U}UIG|Gkc z-qaq=b6qXI0Vuhu^ysZ)kDFVE*@q2fOXmA&!9GQ_iRkN>~+P3Y=%wcCLog-Z1zVD@Mc*_1?i_SIfbMXceFno<4;#ELHE$T zJ95C+o(52Yl8;6MArt za%28gvS$dN^NVEQrEe)IZ<|>3B&>{NtkA;D%L?LB-swYr*JKuzSeV+HGXLm*@c2*@6iL91*@1KS3enm9p7TpAul*`RND!OX#SuOU(+cWzhi037Fh7{f#njjCo|8S&Q);-dRuYqTY7AituCsD) zTEsHx+Ks_jMGO>fOQ0Q7jzcOvR8f`FAm`rJiaKT!1o%U$ypY};>_t=@hpldmqs+1nA?5M&h`><)_g8^6k~}Nmk5c{zLAAjeXH0oKB0;<3_$*V3h^qthef0( zK?rTvt@wC&08&qzbid2B@~CTxP@@mJmA}J!p_$~g=|xQXCy6TPW#cF}TzB5}l8oVR z-yCw}Rua#f!dt?}T!?x!H z*f!-V!!M3eKLv(OQHnYlP^vK1iW-FtcbSp~EuMi^Z9jDvTh9o1n@|{nzk|g$?BjOd zumzLyp~_1!R!*Y!NRur#C%;ZfstV>8VqOun=Xb^oL9$ zvdT!J4!biQC8}zTN^c{;HtVr_2Is45OmfUH)J2) zX_`GJr5mD6yWUbxMV$H@!YhvaK|?cLaK1(}bg~72M*Bt;p!| zY-hdp%8o&nU+09}CfSOdqMd`@Wl3o1kUofp)ih7_!anxab)fgPujx0?UlllNe}?C6 zg-3eTa0NnDwg&PWhQPuj;;aC}upV{9vP-pWj*~LyGr3TpFeTo!VI*!v_l|{EQ{TG} z5xYY;KZlOhKK-7=n@MB3BNk8tcTK`z?PZa({MjL)%z4&R%~@~e=IMKh*_58(C(}{u z^mDDg%d1OatMpdyfFY`rv)vrU)~aoGvZUi-=-w6)cb{ji6HzcM#Y@V~)J;|pqvSUk zoM{ZpNfeA<9I0MWvn|GsBWj77u6)jlkh^^C!A_NHToCF?kM(Z6{JX-rM{M+zf_YO} zbH+ym=!9DjTAgUoRGFe(ErFBO^63NL-Pi7#sn2N^rf?1A%m|cSah3IvMKJZugY*{C zH6~m%CY1J5^Ry%!x8YfAv9LV0uhzxewnK#%g+t5;lXi@;KGdeSMpeL%rb~DwvVTm6 zP%>g>CyjMcU1)anv~sm9X%jDCq=<5)>*+e&{1$$-(u|IcF%b{Z=#si#>rQ0biK-8Hrq~|ZJxQh`stezM2U{nFWylJy za*hH?z=mpt!(&PB8d2Gc1;Va2lIYkrDBc}zv;qwcW_p;IaR10ar+pA>KxQRY{PwN; zrooWKTx-PS%cK|z-bf|4{4|rCF3#|ZkqJe55?sZFjLy6RCjPV%sH--`6>gH5A?N3o zU(Jhp>Gr2x68A#+wvbBCM5n9i3hf>g7@aDtnVOK%XFO$EH>Q0ep*mR@-;J=zYqlzM zN7U3PFV(U$z#=M6bKTgwO7IArR#3kh$R5#h6=FD$$-cQ8T5g>&gK-bw_j2&#`Jl3^ zBx0XRL31}rFp$J$5V@SF84bJE`jVRQ$(Pm!W<_S6F$v$W1NC*FN*-E$X|Nmao8gB_ zoqbn2ly!r6WgXxObQZQZPb&FTbbp0z2wA3vPs4c05yK0)$Yshl9>|JFuib@^7$#YUuegEWA@_ll-q>K9*2knBaBbC6|Mag{ZC%~;#1wEWDFBWok^bX;on1U_ zOr3vRv|npmFR@_yk{rH*ZELAs;fV}9s^;LtY-W{$d8=mg5^ai7O2&cfYR%n*FI=>r zYZ2#Q_hy%Gt;x*29CP<`)V9_(Pcs~=QOP8ydM2Tt4^tqX#H_NKY97|`X}qSldaGZm zraPuho#_afzY2;sk)X{O{*hx*YJc5|tLGT;ecfej0jVRdLU@7B2C*Si z=@Uf!wD;NxP-d9O)f)uu-j9qxp$Q`4mLX`y7OIP;sui*2;AFJ|4N8CT*^oRNvM42R zhiVk#M*1`_+e|ABSI#8mf6rS$LuTlW_10Q1oP#WcsF9h3sak^GT)FUiNSGByJbr;> zR|wjITrQ$)QiNbDS0=7N;eD}5Tdx3?w{0W0b@A5be5vuLU3k8EX=KYQnN3~f&|pcO zF!q3X41jpNa3LXkNe}}5*3K8?WqOMe;-Z@Gp_hH*#+cOaVG5izOss9KEH)i?+e!wW zSrejide>ZjnBu#ssB>1@B`|54#S;DM%V)p&u{UBH6zB^6`*bRfrn})uL^u%kdJX%U`o0 z6*H(Ea#T?>HLC_^Kn#T6jB|W@&)$!7i&1*3*JobJe4nce6aOW0S-Wr*#(E_&_zH+| zgfwxCZ};pgJEya zhN4juX!hMBso#wdrOG+DmdtzRkDr68u~I$Ss(%I}9eS0+`;I ze-hZj(8<(9)y2uu&in_DP2xIbpoGC0HQ))$$jazqVAzNpEp%J5$R)(&N+VxmVp*b1 zmHkOr*tmeN{u##EcaS?2G^Z4!d-$Efn%=t;;&L?TeFKv&17;dG414?5 zu#5~!I5(y2oUBB*&xqBVa32*`n@q+d3fuReVHv~-1VsgkQbK(U3PABrMJks535v*j(KH4ns*IjUT)=m2(g zxYxzrfR*$2;Q5vgE{ymsYdPfi28H}}u# ztZQ%(CwN-0xARf)v>X=akF=I;E6$wb#Z}&v&T8sJ3P?CEct6%c>IWitU}W_RJvZ)C z;F?{Z+YBF=Q<3oKQs{-IXGn}?a*4nVDZhZvnZY(y!_+qoEvO}@gy~)$i!rL6X3@UW zwUC5DLf1YVS4rB=Pl91I+}&(JqHr!aL_88W;fQ{FSWXsSbH|yXIe-0-uODOl?Y{8d z=i$Ijhm+fF0A7iM9*Gmrs!DUPJ5k^9dWRXZ@C-4{-#^ef|gmEFPH5>%{QL4qNX7(cwXZ>TK zWQ&q~x^ts+b)3x2e5MrF+ik=7`vN~HN~~tfqFBD1nVANPe(pdselhVzSHndVGmh0_ zO2GtemD)`7XV(gQHwl@3;4}E0z9gBtktg7p2rz|F|4v^+2Z#T$7rZflwye0H6#<%S zPCO}$j7hmKvN}{eQ7iAKkI1$7WV)sw-y&i_J8qeMuOsQGUx*TAR)}*VE!(c1Q~`Kd zLAup>ue@TiD;VfC{Ng22y!}km?r=NUQyP;j8KB7m>C6~g*i)ynKw{|7WQ3<_NsCnB zkijc%OTfRJ=tlyGY@YNgG+^*Y;)vvd+D>m2Onwp}ID7YaX81~+%Lc^NC+MMmBh!K) zs9aPZ7CIryoKuAYEAep5$`@J@XcC?Ao)162J6zWJqUWY(%D#aK3c=5big@%>-LP8W zr@F_*GhO!mo$xEL75fiKARrmR#{A!F*?(=yU*o@2wv}Z6Dd3+)x4(dYjA`J%{96g` zSKzOOh@a3laJ=+OS>jjlKQp^Oq2Rg(+z;^oC(rw literal 0 HcmV?d00001 diff --git a/excel2json.js b/excel2json.js new file mode 100644 index 0000000..2797060 --- /dev/null +++ b/excel2json.js @@ -0,0 +1,102 @@ +const XLSX = require("xlsx"); +const fs = require("fs"); + +// do export for excel to json output or data json object +function _internalExport(props = {}) { + const NAME = process.env.APP_NAME || "excel2json"; + console.log("APP NAME =>", NAME, "\n"); + + // load from env + const INPUT_FILE = + props.inputFile || process.env.INPUT_FILE || "./data/people.xlsx"; + const OUTPUT_PATH = + props.outputPath || process.env.OUTPUT_PATH || "./data/outputs"; + const MAPPER_FILE = + props.mapperFile || process.env.MAPPER_FILE || "./data/mapper.json"; + const SHEET_NAME = props.sheetName || process.env.SHEET_NAME || "Sheet1"; + const ENCODING = props.encoding || process.env.ENCODING || "utf-8"; + + // get mapper in string + var mapperString = undefined; + try { + mapperString = fs.readFileSync( + MAPPER_FILE, + { encoding: ENCODING }, + (err) => { + if (err) { + console.error(err); + } + console.log("Load file suceed =>", MAPPER_FILE); + } + ); + } catch (err) { + console.error("read file error", err); + } + + // convert mapper from string to json object + const mapperJson = mapperString ? JSON.parse(mapperString) : {}; + const configs = { ...mapperJson.configs, ...props }; + console.log(configs); + const columsData = props.mappings || mapperJson.data || undefined; + + // read workbook from excel file + const wb = XLSX.readFile(configs.inputFile || INPUT_FILE); + const xlData = XLSX.utils.sheet_to_json( + wb.Sheets[configs.sheetName || SHEET_NAME] + ); + + // mapping the data from read excel file + const data = xlData.map((row) => { + if (columsData == undefined || !columsData) { + return row; + } + + var _r = {}; + + columsData.map((col) => { + _r[col.label] = row[col.dataIndex]; + }); + + return _r; + }); + + // able to save to output or not (default is true) + if (configs.saveToOutputput) { + // parse a new path + const outputPath = configs.outputPath || OUTPUT_PATH; + + // check directory and create it if not exist + if (!fs.existsSync(outputPath)) { + fs.mkdirSync(outputPath, { recursive: true }); + } + + // json data output + const jsonStringData = JSON.stringify(data); + + try { + // write to file + fs.writeFileSync( + `${outputPath}/${ + configs.outputName || "exported" + }_${new Date().getTime()}.json`, + jsonStringData, + (err) => { + if (err) throw err; + console.log("Write succeed!"); + } + ); + } catch (err) {} + } + + return data; +} + +/** + * excel2json + * Allow to export data from excel to json object or output of json. + * + * @returns JSON Object of result + */ +module.exports = excel2json = (props = {}) => { + return _internalExport(props); +}; diff --git a/index.js b/index.js new file mode 100644 index 0000000..ffd8463 --- /dev/null +++ b/index.js @@ -0,0 +1,16 @@ +// import excel2json module +const excel2json = require("./excel2json"); + +// called function export excel2json +const exported = excel2json({ + mappings: [ + { + "dataIndex": "Name", + "label": "Name" + } + ], + "saveToOutput": false +}); + +// output data from exported +console.log("Output =>\n", exported); diff --git a/package.json b/package.json new file mode 100644 index 0000000..984d7d3 --- /dev/null +++ b/package.json @@ -0,0 +1,15 @@ +{ + "name": "excel2json", + "version": "1.0.0", + "description": "Excel to json", + "main": "index.js", + "scripts": { + "start": "node index.js", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "author": "Sambo Chea ", + "license": "MIT", + "dependencies": { + "xlsx": "^0.16.9" + } +}