From 43428d159ae225e55b0fdb6a63ea13c8095a6efe Mon Sep 17 00:00:00 2001 From: Mo Eseifan Date: Mon, 20 Apr 2020 12:32:11 -0700 Subject: [PATCH] Added decrypt Refactoring to reduce duplicate code Some cleanup Addressed comments --- docs/SensitiveRecordDecrypt-transform.md | 30 ++ icons/SensitiveRecordDecrypt-transform.png | Bin 0 -> 8529 bytes .../plugin/dlp/DLPTransformPluginConfig.java | 165 +++++++++ .../plugin/dlp/SensitiveRecordDecrypt.java | 224 ++++++++++++ .../plugin/dlp/SensitiveRecordRedaction.java | 220 +----------- src/main/java/io/cdap/plugin/dlp/Utils.java | 114 ++++++ widgets/SensitiveRecordDecrypt-transform.json | 337 ++++++++++++++++++ .../SensitiveRecordRedaction-transform.json | 3 +- 8 files changed, 884 insertions(+), 209 deletions(-) create mode 100644 docs/SensitiveRecordDecrypt-transform.md create mode 100644 icons/SensitiveRecordDecrypt-transform.png create mode 100644 src/main/java/io/cdap/plugin/dlp/DLPTransformPluginConfig.java create mode 100644 src/main/java/io/cdap/plugin/dlp/SensitiveRecordDecrypt.java create mode 100644 widgets/SensitiveRecordDecrypt-transform.json diff --git a/docs/SensitiveRecordDecrypt-transform.md b/docs/SensitiveRecordDecrypt-transform.md new file mode 100644 index 0000000..e50061f --- /dev/null +++ b/docs/SensitiveRecordDecrypt-transform.md @@ -0,0 +1,30 @@ +# Cloud Data Loss Prevention (DLP) Decrypt + +Additional Charges +----------- +This plugin uses Google's Data Loss Prevention APIs which charge the user depending +on the volume of data **analyzed** (not transformed). More details on the exact +costs can be found [here](https://cloud.google.com/dlp/pricing#content-pricing). + +Permissions +----------- +In order for this plugin to function, it requires permissions to access the Data Loss Prevention APIs. These permissions +granted through the service account that is provided in the plugin configuration. If the service account path is set to +`auto-detect` then it will use a service account with the name `service-@gcp-sa-datafusion.iam.gserviceaccount.com`. + +The `DLP Administrator` role must be granted to the service account to allow this plugin to access the DLP APIs. + +Description +----------- +This plugin decrypts sensitive data that was encrypted by DLP using a reversible encryption transform, such as `Format +Preserving Encryption`. The plugin works by reversing the encryption specified in the config. Therefore, you must provide +the same configuration properties that were used to encrypt the data. In other words, the configuration in this plugin +and the DLP Redaction plugin must be identical for the decrypt to function correctly. + + +Metrics +----------- +This plugin records three metrics: +* `dlp.requests.count`: Total number of requests sent to Data Loss Prevention API +* `dlp.requests.success`: Number of requests that were successfully processed by Data Loss Prevention API +* `dlp.requests.fail`: Number of requests that failed \ No newline at end of file diff --git a/icons/SensitiveRecordDecrypt-transform.png b/icons/SensitiveRecordDecrypt-transform.png new file mode 100644 index 0000000000000000000000000000000000000000..1f4e1547a71adb366332b249591ebe4513726afe GIT binary patch literal 8529 zcmaiaWmr_-7w(y1fEi+dq3Z|IAf+HF4N?jSBHcg5CH%{s;;K23jh%C76QNtz>BFz zsV#UxxTzU=0ssx?%@3lk%draptbn?*!V|yg?VKPd(;pe#dD1KyQ9WOi@)PzHt1G6T zSbeE3L8;yH-d~LPO2Olx_>Q_lth#0LA)3xu_?gYL#vcN^xs)<4G5szU_1;j8 zCE(sER+EG84svF4+P2Ooze$7(KR$;(fNLoOCl}b0K_77cI5%Ktw(6K$1-2VwHzczr zIV`yo@T;q6+<_csrNuu1gDF7e$zcdUft&+?DI@^o|DT3+BF1|Pf{$_@i(^PH@(6&< z5Bgq%oPZJ_2(6#m@r?MO?$R~t(B2Tz29cM>I8QJ{uHFM$E@gkJydXkh2HJP>Gkzrx z<#%lgziOsU0Lz{e+8ltxa5%qrX1@A~KffaD)e$pLWKs5Hu7HZD%LQ>#j;uuc4xP9CZ z*$YW_IpT(IB-v5|MCTL?&t>Mt)d4D|3zy7Gga>5R8}OQrAGxa^Xm2dOZcCZ4aZV|v zU|^*nBZ)yWT!bS_N5*yKX_1)kffQ28v{ni&5*Xx$cKFap1o22SFtRNht~IJ+s57n! z$Zm|cTdjq29ln<1W1m!)U?XNC3KD3b&*vQO(aX?^u7fSA0gufj%HPmrL=7$l; zSy_tG(oDqQc{6Nz~1MXd_yp!sPg!vEK-C`v&y5sjCR%M)hIbc2);1Lzu} zm2*nlLUUp(1^qMtH0Hy&jg_tmAAlJ=a!Nf&N>^x^B+xkbigmU0@ooOnV!Zr7bg0ft z{!%u;tBFiu85>I$k<|?u+0rI&ILK54PO|iPwp7U*p6uP)OVk~EBoA)*>$~Lp{DvH^eGUpH2!ggO+v_pCrthI=e79d17-7pomA3haO_5q7j)i-oP#fNuM*~;; z8;2a=?Td-ACd`XDY48}1*<`d=?2f!Gstc~nZ%@wIrCeTu{Ua+&c4y=U7r_3K5o5bg z+J?QKe1W#7oiaeTFK#O~By}~Omrpg@`Yfa+e$3<88|o49TU(;{C;wjHXQ=asQ=X7C z__^(Q4%u||%h*TQinOe=fQr(#pV60OLLH1OLkl_al*>fR+hLvME^aM5x*!AB2I&=)Elk0zg%2^KZ>>Tn|&G`DP7yY{_1jNG(Ug>n$sV24=wd zLfvwofzdp8)}g+G3=kxIl^56_HR$MckdJ$N`ScduNOHZu@{Obdo%7Dtcj6M zRtl2L@DQWl1?k+2bI$0iIC-8B zdEpRCKLRCocR=3lew2=%4^8FA+QL~$XgZ*L} zw&5Qi8v&gu?di%be*l$)$6kS2m^fR?< zCE0c5WNuKCg03k_?m9(`P(?Qamk{vk#f?upB}+>-x1v8dHC>Jx5}B=WjjY9|F4UM{ zHw!$BB3P-ke&?aZqCRABPDp;e)6s-^JSm=V1`GH@|9WkuXXVX)UIVJ5lD#02Bp7)8 zRd@X*8r2jR?zXC-oO*mvU*UIjhg(;~eS_(Df~j&c7*D3+V`XABcjIHnd@qz1IdajL zRrNFvOV5Iu?;-Dhv5DYkjI7FL?rCb%?y~mOQKcaCJQe0eVr(+cS|f*>){A4t(B(z_?q9wbQZ!u_p!4&Tw2N5nt^2UH*{wF7-YkQim8 zs=&qyFUIEc5ROdHI|B>im&R9u$dKXiFIf|~>t^%|7t5%k%CWf0$&^qfpAa3WbC0W< zxwV=%l%5uluch35GC-qgW?y%-H-?T6u_vowQNyFMbZH7yMnNSVPpP3UMP{hEMuw=XE5KBDt3j6g@nzMGUpx& zQDqXva=-D3v19S1s%X8f411rAmUFlA_}h*o2E9h4tq!-D$N>Ou0_p8h#7Q6|Y$>sy zeV6O=BBq-Z+EsWIRr1A#&sgre7v|SHn8Md*VyBToyPGd2qX)A z=C8-`D#J{NX6f2lo;vG!z(`h*t1QwzK9++Z{M|$JUgNf9L{UBa zRp_{iSJz~%KjkdEMrZ;jjY4JCP+ecp-6l)q98fjK4cYL*X3w-OKq@~8YjO0~8T19S zB4*Ni$Yf~&IE<2xE|+Et7JM@K=wc^UZOGraoYL@^y6y(F8Y5eUqPZ*o$}=w6-AQ7* zX|ppT>M;?Wc^OBoXS9U6B20-0_t3Tk5WUMouguP$t_H5|U-4%9p%i|)VQKKZC0Rz+!~l7FSSm1XNeD#5F4!;4G1RZ=zk%->Au?LEIb zxFNsK=rA8UR46-Y(PU2VCRJVZ?>Pjnt&RJgrPW*uFletyK$y1?PD?q>Zjc3WGOv5* zc3*I15x54XxBJFv6W66gC}Wns4z3^SJ1w>;o>TE3nSEc( z*1MZ?IDG1Yj_Lag{3E<;M=}2w$3~v1wv-cZM^4w&a)EgCes4yOQzeKiHcGzAkTCxZ z)?#reFZXx{N2b$@JA5hK%_i<+IMhQH+x^_k*Effl{rhfkh>oHdcj6G>?&r zY(91@fDI_hcg=qWjl_bHd63-(o#YGN(!H0KQ1}8^6EQ5Rug=ZjMbOoL*$EvrX z1wuL&&lPg)`m&i>=yD&-tLhDewe1wn{AnoI`zDUxpJpsi6}dtWr~Z+9%*~ls2!@VS z0j7_Sxkui*L+Z+L^TH~!s;zFU!c@4GParI`?O{eXocOo_QTnB|_L z3Zu%tvtecRL|B)Ax-Lt@w0ew*Jl{)A+B}0O(#q~1nxpmpA4=cI7FJUcMB0Ev!CT0m z1N!C@ZMzC13*)TqSAxkTxSu1_`df$M`AZ$i%$Da54*^px;yS;6>^GChx(aFuZ%(!u zv#{`DWlJ(lix~UR)r&Xs&l(THI6dtw`n#WhYANY=PylbHTI!feDwR2^Dh?-$AJB#r z53Tz!eo};TKqCkZR_WZ`?i+dK1PUbUY)NfLlzyyT0uyOEu^SOJ)HN2bHEB$U(+cqi z5u5nzlQ4UAGM9_iSUubc3n>-bXg_}9Hr3o6H|SUnkDG~3g`4zheQ_}s+c_}c$^LrV zt}VfP&b&v$FRMkc*100#@cUmykY~7^(UVPU%_fM}FVmDMiqojt@MhNw^YXc%t*^`8 z<+<963N{KkJb^Jdjo{wS|IV>m{5R6!zUTNS_tIBD0(;-(wz;#J(Qfg7sq|wjwF4i~ z+)k?&=3__Rk^<{5%%aRx6`dCEck4cXb6Hs#beoo`R)SObH0BpBpY2f0r11*Yqs^4r@P5j(tQW+JR>)OT33R=GpNV(H{f zK>YcSBEITAWh{6M>16gFc%htXG-X(Nb%5yCdZ#6CjWhZv#i;2&zP22pA=-)oUr2|0 z-FI;?6Ls_x#m{se$lB94wYK*RP zZdD6hB1GS5rW7{x`L?am3KBm%Ey=+hI9^w^B(eWlm|7gAz3F!LvH#?=%m);VuQc-F zY#g|6VV&ZJ8mlt!RfMJ6tn4{tk3Fy6N`N4HBg^dpbxI3+k%~rhhhASD_~1UQR#neE zO0&?FCU3mwim{V2FCe*>%2)sSKk54}S++HM?-S)LBXoJ72|}Si|K;ile`JnuO0>08 zCs0IysdaZ!LdV2Gl~uyzWA(%-xrQ`E3DKT+{UYp#xtPISV?jfoX1orV3H{c2PV|$d zQD_s3b<+p6^-pR;Y{lRbLo(Om``K!1ICqs{M6xlA#P>x+(d!ZvHWKbz(Sr9N;-0v5 zsj@Ez74f&mY#OHE-&*=&R4K+tL%@`cVk8dmn|*4wZAe@MvXf8Oy3@J`JM^)`+l&#N z>e(RSuWsFKUe^YBSXpfWl+A69z>!+mPRgSBP!B)Kg?PCP2f$Y6#fF#=3Q%h#TqMsrPmwpRMwr6VZ+eJige$wFgq zh-LOv@rl6)^Uj1AEz4f$!XxrGiFsS>qRs{yf9`xv3{19{p2VS#?ElsN9_HOQ1|#WT z?W+L%eKwI7d2z|=uwS)@=cbFGN16z)zqRBtVv53JbiEGn`4r#d+>hWDsM}|XDiW5s zKUjOJzgb=QeEd@=_iiOVCH_v85Vhhv_KK`j^~_6JM*GtMJdn1n!XG%VY?mwDlDt(_ zSDAf&{_^^X);m3)xCc+z31c|~2`i7!O_ef)7qU12(95Z`hP-FNP_oPag^m2-K-OUN zpiW7Za=LGuiI`y5!tOz6j=;P2A7&m)74FMaQ&cTnv~DR=FRv#|Cv8{rz>3cJUwFxs z=Xcsou;MT!o>OE^PZual{G3);J3|)!2>&u25(L0!8shhmGiOc4>@2T!+TKM( z$$(G`YOrNHOFxFS%dz;| zc3Q>WvhF2(F6~Qo1>kaE;oQKFuTDta8rOforo;L@tp<@!iZ(NhhnOfI#&$eRe^?1k z9yKg`M(amamdqcaBXLv1WnQLap10>Td#(R5E z*J#OCO%7J{qLt>+RaNHSvxhBW6pMZ!aJJW7X~?3&;}u&nZWVPne23swDEvWxa-5-dti5<_dkfh=8^` zrN!`P1eip&1rOs9o$Y#=Qt`6+1JM)B&hc--`K8CA&kWrl6gnbOps4Yin!Yp4KZF3S zPS+f`o-2}wkLmHVQXq|Pwjmi&N~ zL342-iV>W8G^Hb_8Qi_Lnp&8{ivYg}#V@Z}M&1FSJ8lxzB1vM+4t>%p3wNb56yvC=$+4+NNA#0fjK2+`IfWEB} zl7|<%qD8DFw?vVRx-S7LPyDf<;-oV6+F@a{I?MQ2xJ7W#Q1U0X1Yh3`>TEI>+HVJv zidrvpq>di}I5M;CX(V`jbx9GSC0UCO#nZ3Dp4U8q$~;V&Pyz9SHaC zY#g;y=i_Zl8X&We^d7}$jUeJalimx5j>2Ncz^Fb&0hHjN=-7`%m`GUK^`U0lV>7@n zuJg4oF&_sMj6hACrkRq$Ooe}4Jq){oqu~JB7~Az;Td@`n(lg7rc~zSaFKMk&T$>&X zNDKdlf5Vv+pv(DN9gK(GeJ#&59-pFTz*hU%&IJTbT<@W+N6B07^rLDx6=X72&QgpJ zj6Z#<(O349pg_UmC|@EqIv#%So&yq}UaF zXsr0_3?Ca%j3gFhg~}or(VzyuOmd#`9g;51pZ+b^RRXKg*=_N&0U`C$&gB{DdCEh; z@8LESRLFf~dxK}K?M1cLwh=~M!JO1U%`5!8z$r*tm`$)u#O1@`{8_6u-TxWh%n;l9 zJ=*)Y+T>cqfX&NecTo(-T1q{Zr%(eyk2_e+VkKVplv%!v{6jcZ$H?lE8R4{;{aw+uA)D9G zOqk#fi6ka=xspa6>Tjiaj&!p;j>hI~KrJV*mQpznlHGxEu&E52M+4fi9lOwk2CfPm zp>~z8zwFiS+4MyKaz)mrCN<25lP+YbWY(PZI04Y*13 zTe2wS;fOx`W5V@PA;^M*{@klfB*qugBD_8qO3w`dpqqkvMD;fl0stP4vfus>M5zk_ z2%3_|Kl(KAVn>i+xikY3kpD@z_s{WzC4?Jy5*69z*b2gr~IVu z0vib(&HEMLZA6)soPadY)dYAzVPk)uJ~~tt2ucsmbA)s8uxYSa8Wa5JJ^q=5*ISQ4 zpdg)Yj_9GXhCtdu1c26Rr&3W?hdmTM>F`b~8%X=bj9Kgo<$xC3pt4s8Gq=zTF10>2 z2S^m8%Tw*PRolxUWlgL0zyWfafk?&hNM$`MIT=YnuK`?{K*h#N&J2Mu>IBC}s(4z- zwMc@YAf1(h^&pa_E4Yyo5Q)H7hmnA5F;7}gX~!empI9lNAWFhU-)DkIga9K=NCKQ( zYJt8Y@^V#@THqCA|Bi{FFe4GgM&!H&n5YcAMLO2Y-Dd`;KlUQl!@bUU%?@wN1R07% zg7Y5W5&r#YOm_Zf4)V;j9To)?$lzmZ|5VET*~ZJd$FIhge~Po9wUT3(0LE|ni@tH` z42~qnlotVNFg2Jc2K>99{U)~)bDA#6pH3P9*H<~52^!b{dhrt|b@IHpN z@h3f}*9AVMLSa$zynaSipgpx741@OM;g?_`qWB(H*%b`Fg$1x_XWWtymC=_|!z0j} z_qb5M6o1|*44V%Qu#zj327vYdO}T6bo)Tv9EMl~koP}m9|4M_6b$<&zz{7}^obOl5 z!r;KS#Hx@dkALQnIg>;`82@sffM=NAGU3{X=>0k#=`Hra8)zo3hZ7MeQYJ(ebOzgc z0L#!`gj(J$$xQ{NV+cE1QkMS&t(lb%N0@%O^u`JRjwG_|=J7l?bV{XK4o`{#n;BhDH?xGiXTWFPk1{&!LA3%qp6 zXQ+Ap?I@95xSTS5x&4wTdj-E{>t7Opjp`r(W?uS^$cN@TQqV28n>}I=+WS`!Ff^ja z0ZHH^39m!!Gq(KibAdNj1zfCf6cEa)2EapEA*}FF80P=cFv5OKX0+!? parseTransformations() throws Exception { + String[] values = GSON.fromJson(fieldsToTransform, String[].class); + List transformationConfigs = new ArrayList<>(); + for (String value : values) { + transformationConfigs.add(GSON.fromJson(value, DlpFieldTransformationConfig.class)); + } + return transformationConfigs; + } + + /** + * Get the set of fields that are being transformed or are required for transforms to work. This is used to limit + * the payload size to DLP endpoints, the transform will only send the values of the required fields. + * + * @return Set of field names + */ + public Set getRequiredFields() throws Exception { + return parseTransformations().stream() + .map(DlpFieldTransformationConfig::getRequiredFields) + .flatMap(Collection::stream) + .collect(Collectors.toSet()); + } + + public void validate(FailureCollector collector, Schema inputSchema) { + if (customTemplateEnabled) { + if (!containsMacro("templateId") && templateId == null) { + collector.addFailure("Must specify template ID in order to use custom template", "") + .withConfigProperty("templateId"); + } + } + + if (fieldsToTransform != null) { + try { + List transformationConfigs = parseTransformations(); + HashMap transforms = new HashMap<>(); + Boolean firstTransformUsedCustomTemplate = null; + Boolean anyTransformUsedCustomTemplate = false; + for (DlpFieldTransformationConfig config : transformationConfigs) { + ErrorConfig errorConfig = config.getErrorConfig(""); + + //Checking that custom template is defined if it is selected in one of the transforms + List filters = Arrays.asList(config.getFilters()); + if (!customTemplateEnabled && filters.contains("NONE")) { + collector.addFailure(String.format("This transform depends on custom template that was not defined.", + config.getTransform(), String.join(", ", config.getFields())), + "Enable the custom template option and provide the name of it.") + .withConfigElement(FIELDS_TO_TRANSFORM, GSON.toJson(errorConfig)); + } + //Validate the config for the transform + config.validate(collector, inputSchema, FIELDS_TO_TRANSFORM); + + //Check that custom template and built-in types are not mixed + anyTransformUsedCustomTemplate = anyTransformUsedCustomTemplate || filters.contains("NONE"); + if (firstTransformUsedCustomTemplate == null) { + firstTransformUsedCustomTemplate = filters.contains("NONE"); + } else { + if (filters.contains("NONE") != firstTransformUsedCustomTemplate) { + errorConfig.setTransformPropertyId("filters"); + collector.addFailure("Cannot use custom templates and built-in filters in the same plugin instance.", + "All transforms must use custom templates or built-in filters, not a " + + "combination of both.") + .withConfigElement(FIELDS_TO_TRANSFORM, GSON.toJson(errorConfig)); + } + } + + // Make sure the combination of field, transform and filter are unique + for (String field : config.getFields()) { + for (String filter : config.getFilterDisplayNames()) { + String transformKey = String.format("%s:%s", field, filter); + if (transforms.containsKey(transformKey)) { + + String errorMessage; + if (transforms.get(transformKey).equalsIgnoreCase(config.getTransform())) { + errorMessage = String.format( + "Combination of transform, filter and field must be unique. Found multiple definitions for '%s' " + + "transform on '%s' with filter '%s'", config.getTransform(), field, filter); + } else { + errorMessage = String.format( + "Only one transform can be defined per field and filter combination. Found conflicting transforms" + + " '%s' and '%s'", + transforms.get(transformKey), config.getTransform()); + } + errorConfig.setTransformPropertyId(""); + collector.addFailure(errorMessage, "") + .withConfigElement(FIELDS_TO_TRANSFORM, GSON.toJson(errorConfig)); + } else { + transforms.put(transformKey, config.getTransform()); + } + } + } + } + + // If the user has a custom template enabled but doesnt use it in any of the transforms + if (!anyTransformUsedCustomTemplate && this.customTemplateEnabled) { + collector.addFailure("Custom template is enabled but no transforms use a custom template.", + "Please define a transform that uses the custom template or disable the custom " + + "template.") + .withConfigProperty("customTemplateEnabled"); + } + } catch (Exception e) { + collector.addFailure(String.format("Error while parsing transforms: %s", e.getMessage()), "") + .withConfigProperty(FIELDS_TO_TRANSFORM); + } + } + } +} diff --git a/src/main/java/io/cdap/plugin/dlp/SensitiveRecordDecrypt.java b/src/main/java/io/cdap/plugin/dlp/SensitiveRecordDecrypt.java new file mode 100644 index 0000000..b2902bd --- /dev/null +++ b/src/main/java/io/cdap/plugin/dlp/SensitiveRecordDecrypt.java @@ -0,0 +1,224 @@ +/* + * Copyright © 2020 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.plugin.dlp; + +import com.google.api.gax.rpc.ApiException; +import com.google.api.gax.rpc.ResourceExhaustedException; +import com.google.cloud.dlp.v2.DlpServiceClient; +import com.google.cloud.dlp.v2.DlpServiceSettings; +import com.google.common.annotations.VisibleForTesting; +import com.google.privacy.dlp.v2.ContentItem; +import com.google.privacy.dlp.v2.CryptoReplaceFfxFpeConfig; +import com.google.privacy.dlp.v2.CustomInfoType; +import com.google.privacy.dlp.v2.DeidentifyConfig; +import com.google.privacy.dlp.v2.DeidentifyContentRequest; +import com.google.privacy.dlp.v2.DeidentifyContentResponse; +import com.google.privacy.dlp.v2.Error; +import com.google.privacy.dlp.v2.FieldTransformation; +import com.google.privacy.dlp.v2.GetInspectTemplateRequest; +import com.google.privacy.dlp.v2.InfoTypeTransformations; +import com.google.privacy.dlp.v2.InspectConfig; +import com.google.privacy.dlp.v2.InspectTemplate; +import com.google.privacy.dlp.v2.Likelihood; +import com.google.privacy.dlp.v2.RecordTransformations; +import com.google.privacy.dlp.v2.ReidentifyContentRequest; +import com.google.privacy.dlp.v2.ReidentifyContentResponse; +import com.google.privacy.dlp.v2.Table; +import io.cdap.cdap.api.annotation.Description; +import io.cdap.cdap.api.annotation.Name; +import io.cdap.cdap.api.annotation.Plugin; +import io.cdap.cdap.api.data.format.StructuredRecord; +import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.Emitter; +import io.cdap.cdap.etl.api.PipelineConfigurer; +import io.cdap.cdap.etl.api.StageConfigurer; +import io.cdap.cdap.etl.api.StageMetrics; +import io.cdap.cdap.etl.api.StageSubmitterContext; +import io.cdap.cdap.etl.api.Transform; +import io.cdap.cdap.etl.api.TransformContext; +import io.cdap.cdap.etl.api.lineage.field.FieldOperation; +import io.cdap.cdap.etl.api.lineage.field.FieldTransformOperation; +import io.cdap.plugin.dlp.configs.DlpFieldTransformationConfig; +import io.cdap.plugin.gcp.common.GCPUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Class for the Redact DLP transform plugin + */ +@Plugin(type = Transform.PLUGIN_TYPE) +@Name(SensitiveRecordDecrypt.NAME) +@Description(SensitiveRecordDecrypt.DESCRIPTION) +public class SensitiveRecordDecrypt extends Transform { + + private static final Logger LOG = LoggerFactory.getLogger(SensitiveRecordDecrypt.class); + public static final String NAME = "SensitiveRecordDecrypt"; + public static final String DESCRIPTION = "SensitiveRecordDecrypt"; + + private StageMetrics metrics; + + // Stores the configuration passed to this class from user. + private final Config config; + + // DLP service client for managing interactions with DLP service. + private DlpServiceClient client; + + // Required fields that need to be sent to DLP for the transform to work, this variable is used to cache the set + // since it will not change during the execution of the plugin + private Set requiredFields = null; + + @VisibleForTesting + public SensitiveRecordDecrypt(Config config) { + this.config = config; + } + + @Override + public void configurePipeline(PipelineConfigurer pipelineConfigurer) { + StageConfigurer stageConfigurer = pipelineConfigurer.getStageConfigurer(); + config.validate(stageConfigurer.getFailureCollector(), stageConfigurer.getInputSchema()); + + stageConfigurer.setOutputSchema(stageConfigurer.getInputSchema()); + super.configurePipeline(pipelineConfigurer); + } + + @Override + public void initialize(TransformContext context) throws Exception { + super.initialize(context); + metrics = context.getMetrics(); + client = DlpServiceClient.create(getSettings()); + try { + requiredFields = config.getRequiredFields(); + } catch (Exception e) { + LOG.warn("Unable to get list of required fields, defaulting to an empty set.", e); + requiredFields = new HashSet<>(); + } + } + + @Override + public void prepareRun(StageSubmitterContext context) throws Exception { + super.prepareRun(context); + config.validate(context.getFailureCollector(), context.getInputSchema()); + context.getFailureCollector().getOrThrowException(); + if (config.customTemplateEnabled) { + String templateName = String.format("projects/%s/inspectTemplates/%s", config.getProject(), config.templateId); + GetInspectTemplateRequest request = GetInspectTemplateRequest.newBuilder().setName(templateName).build(); + + try { + if (client == null) { + client = DlpServiceClient.create(getSettings()); + } + InspectTemplate template = client.getInspectTemplate(request); + } catch (Exception e) { + throw new IllegalArgumentException( + "Unable to validate template name. Ensure template ID matches the specified ID in DLP. List of defined " + + "templates can be found at " + + "https://console.cloud.google.com//security/dlp/landing/configuration/templates/inspect"); + } + } + + List fieldOperations = Utils.getFieldOperations(context.getInputSchema(), config, "Decrypt"); + context.record(fieldOperations); + } + + @Override + public void transform(StructuredRecord structuredRecord, Emitter emitter) throws Exception { + + RecordTransformations recordTransformations = Utils.constructRecordTransformationsFromConfig(config); + Table dlpTable = Utils.getTableFromStructuredRecord(structuredRecord, requiredFields); + + DeidentifyConfig deidentifyConfig = + DeidentifyConfig.newBuilder().setRecordTransformations(recordTransformations).build(); + + ContentItem item = ContentItem.newBuilder().setTable(dlpTable).build(); + ReidentifyContentRequest.Builder requestBuilder = ReidentifyContentRequest.newBuilder() + .setParent( + "projects/" + config.getProject()) + .setReidentifyConfig(deidentifyConfig) + .setItem(item); + + // Automatically generating inspection config using the surrogate types defined in the transform + InspectConfig.Builder configBuilder = InspectConfig.newBuilder(); + for (FieldTransformation fieldTransformation : recordTransformations.getFieldTransformationsList()) { + for (InfoTypeTransformations.InfoTypeTransformation infoTypeTransformation : fieldTransformation + .getInfoTypeTransformations().getTransformationsList()) { + + // Only CryptoReplaceFfxFpeConfig has a surrogate type so target that config, no other configs should be + // possible in this transform since they are not included in the widget json list of options + CryptoReplaceFfxFpeConfig cryptoReplaceFfxFpeConfig = infoTypeTransformation.getPrimitiveTransformation() + .getCryptoReplaceFfxFpeConfig(); + if (cryptoReplaceFfxFpeConfig != null) { + CustomInfoType customInfoType = CustomInfoType.newBuilder() + .setInfoType(cryptoReplaceFfxFpeConfig.getSurrogateInfoType()) + .setSurrogateType(CustomInfoType.SurrogateType.newBuilder().build()).build(); + configBuilder.addCustomInfoTypes(customInfoType); + } + } + } + configBuilder.setMinLikelihood(Likelihood.POSSIBLE); + requestBuilder.setInspectConfig(configBuilder); + + ReidentifyContentResponse response = null; + ReidentifyContentRequest request = requestBuilder.build(); + try { + metrics.count("dlp.requests.count", 1); + response = client.reidentifyContent(request); + } catch (ApiException e) { + metrics.count("dlp.requests.fail", 1); + if (e instanceof ResourceExhaustedException) { + LOG.error( + "Failed due to DLP rate limit, please request more quota from DLP: https://cloud.google" + + ".com/dlp/limits#increases"); + } + throw e; + } + + metrics.count("dlp.requests.success", 1); + ContentItem item1 = response.getItem(); + StructuredRecord resultRecord = Utils.getStructuredRecordFromTable(item1.getTable(), structuredRecord); + emitter.emit(resultRecord); + } + + /** + * Configures the DlpSettings to use user specified service account file or auto-detect. + * + * @return Instance of DlpServiceSettings + * @throws IOException thrown when there is issue reading service account file. + */ + private DlpServiceSettings getSettings() throws IOException { + DlpServiceSettings.Builder builder = DlpServiceSettings.newBuilder(); + if (config.getServiceAccountFilePath() != null) { + builder + .setCredentialsProvider(() -> GCPUtils.loadServiceAccountCredentials(config.getServiceAccountFilePath())); + } + return builder.build(); + } + + /** + * Holds configuration required for configuring {@link SensitiveRecordDecrypt}. + */ + public static class Config extends DLPTransformPluginConfig { + } +} diff --git a/src/main/java/io/cdap/plugin/dlp/SensitiveRecordRedaction.java b/src/main/java/io/cdap/plugin/dlp/SensitiveRecordRedaction.java index c9771ed..0d64be4 100644 --- a/src/main/java/io/cdap/plugin/dlp/SensitiveRecordRedaction.java +++ b/src/main/java/io/cdap/plugin/dlp/SensitiveRecordRedaction.java @@ -81,7 +81,7 @@ public class SensitiveRecordRedaction extends Transform(); + } } @Override @@ -136,87 +142,13 @@ public void prepareRun(StageSubmitterContext context) throws Exception { } } - List fieldOperations = getFieldOperations(context.getInputSchema()); + List fieldOperations = Utils.getFieldOperations(context.getInputSchema(), config); context.record(fieldOperations); } - private List getFieldOperations(Schema inputSchema) throws Exception { - - //Parse config into format 'FieldName': List<>(['transform','filter']) - HashMap> fieldOperationsData = new HashMap<>(); - for (DlpFieldTransformationConfig transformationConfig : config.parseTransformations()) { - for (String field : transformationConfig.getFields()) { - String filterName = String.join(", ", transformationConfig.getFilters()) - .replace("NONE", String.format("Custom Template (%s)", config.templateId)); - - String transformName = transformationConfig.getTransform(); - - if (!fieldOperationsData.containsKey(field)) { - fieldOperationsData.put(field, Collections.singletonList(new String[]{transformName, filterName})); - } else { - fieldOperationsData.get(field).add(new String[]{transformName, filterName}); - } - } - } - - for (Schema.Field field : inputSchema.getFields()) { - if (!fieldOperationsData.containsKey(field.getName())) { - fieldOperationsData.put(field.getName(), Collections.singletonList(new String[]{"Identity", ""})); - } - } - - List fieldOperations = new ArrayList<>(); - - for (String fieldName : fieldOperationsData.keySet()) { - StringBuilder descriptionBuilder = new StringBuilder(); - StringBuilder nameBuilder = new StringBuilder(); - descriptionBuilder.append("Applied "); - boolean first = true; - for (String[] transformFilterPair : fieldOperationsData.get(fieldName)) { - - String transformName = transformFilterPair[0]; - String filterNames = transformFilterPair[1]; - - if (first) { - descriptionBuilder.append(" "); - } - descriptionBuilder.append(String.format("'%s' transform on contents ", transformName)); - if (filterNames.length() > 0) { - descriptionBuilder.append(" matching ").append(filterNames); - } - descriptionBuilder.append(",\n"); - nameBuilder.append(transformName).append(" ,"); - first = false; - } - nameBuilder.deleteCharAt(nameBuilder.length() - 1); - descriptionBuilder.delete(descriptionBuilder.length() - 2, descriptionBuilder.length() - 1); - nameBuilder.append("on ").append(fieldName); - fieldOperations - .add(new FieldTransformOperation(nameBuilder.toString(), descriptionBuilder.toString(), - Collections.singletonList(fieldName), fieldName)); - } - return fieldOperations; - } - - private RecordTransformations constructRecordTransformations() throws Exception { - RecordTransformations.Builder recordTransformationsBuilder = RecordTransformations.newBuilder(); - List transformationConfigs = config.parseTransformations(); - - recordTransformationsBuilder.addAllFieldTransformations( - transformationConfigs.stream() - .map(DlpFieldTransformationConfig::toFieldTransformation) - .collect(Collectors.toList()) - ); - return recordTransformationsBuilder.build(); - } - @Override public void transform(StructuredRecord structuredRecord, Emitter emitter) throws Exception { - if (requiredFields == null) { - requiredFields = config.getRequiredFields(); - } - - RecordTransformations recordTransformations = constructRecordTransformations(); + RecordTransformations recordTransformations = Utils.constructRecordTransformationsFromConfig(config); Table dlpTable = Utils.getTableFromStructuredRecord(structuredRecord, requiredFields); DeidentifyConfig deidentifyConfig = @@ -251,9 +183,8 @@ public void transform(StructuredRecord structuredRecord, Emitter parseTransformations() throws Exception { - String[] values = GSON.fromJson(fieldsToTransform, String[].class); - List transformationConfigs = new ArrayList<>(); - for (String value : values) { - transformationConfigs.add(GSON.fromJson(value, DlpFieldTransformationConfig.class)); - } - return transformationConfigs; - } - - /** - * Get the set of fields that are being transformed or are required for transforms to work. This is used to limit - * the payload size to DLP endpoints, the transform will only send the values of the required fields. - * - * @return Set of field names - */ - public Set getRequiredFields() { - try { - return parseTransformations().stream() - .map(DlpFieldTransformationConfig::getRequiredFields) - .flatMap(Collection::stream) - .collect(Collectors.toSet()); - } catch (Exception e) { - LOG.warn("Unable to get list of required fields, defaulting to an empty set.", e); - return new HashSet<>(); - } - } - - public void validate(FailureCollector collector, Schema inputSchema) { - if (customTemplateEnabled) { - if (!containsMacro("templateId") && templateId == null) { - collector.addFailure("Must specify template ID in order to use custom template", "") - .withConfigProperty("templateId"); - } - } - - if (fieldsToTransform != null) { - try { - List transformationConfigs = parseTransformations(); - HashMap transforms = new HashMap<>(); - Boolean firstTransformUsedCustomTemplate = null; - Boolean anyTransformUsedCustomTemplate = false; - for (DlpFieldTransformationConfig config : transformationConfigs) { - ErrorConfig errorConfig = config.getErrorConfig(""); - - //Checking that custom template is defined if it is selected in one of the transforms - List filters = Arrays.asList(config.getFilters()); - if (!customTemplateEnabled && filters.contains("NONE")) { - collector.addFailure(String.format("This transform depends on custom template that was not defined.", - config.getTransform(), String.join(", ", config.getFields())), - "Enable the custom template option and provide the name of it.") - .withConfigElement(FIELDS_TO_TRANSFORM, GSON.toJson(errorConfig)); - } - //Validate the config for the transform - config.validate(collector, inputSchema, FIELDS_TO_TRANSFORM); - - //Check that custom template and built-in types are not mixed - anyTransformUsedCustomTemplate = anyTransformUsedCustomTemplate || filters.contains("NONE"); - if (firstTransformUsedCustomTemplate == null) { - firstTransformUsedCustomTemplate = filters.contains("NONE"); - } else { - if (filters.contains("NONE") != firstTransformUsedCustomTemplate) { - errorConfig.setTransformPropertyId("filters"); - collector.addFailure("Cannot use custom templates and built-in filters in the same plugin instance.", - "All transforms must use custom templates or built-in filters, not a " - + "combination of both.") - .withConfigElement(FIELDS_TO_TRANSFORM, GSON.toJson(errorConfig)); - } - } - - // Make sure the combination of field, transform and filter are unique - for (String field : config.getFields()) { - for (String filter : config.getFilterDisplayNames()) { - String transformKey = String.format("%s:%s", field, filter); - if (transforms.containsKey(transformKey)) { - - String errorMessage; - if (transforms.get(transformKey).equals(config.getTransform())) { - errorMessage = String.format( - "Combination of transform, filter and field must be unique. Found multiple definitions for '%s' " - + "transform on '%s' with filter '%s'", config.getTransform(), field, filter); - } else { - errorMessage = String.format( - "Only one transform can be defined per field and filter combination. Found conflicting transforms" - + " '%s' and '%s'", - transforms.get(transformKey), config.getTransform()); - } - errorConfig.setTransformPropertyId(""); - collector.addFailure(errorMessage, "") - .withConfigElement(FIELDS_TO_TRANSFORM, GSON.toJson(errorConfig)); - } else { - transforms.put(transformKey, config.getTransform()); - } - } - } - } - - // If the user has a custom template enabled but doesnt use it in any of the transforms - if (!anyTransformUsedCustomTemplate && this.customTemplateEnabled) { - collector.addFailure("Custom template is enabled but no transforms use a custom template.", - "Please define a transform that uses the custom template or disable the custom " - + "template.") - .withConfigProperty("customTemplateEnabled"); - } - } catch (Exception e) { - collector.addFailure(String.format("Error while parsing transforms: %s", e.getMessage()), "") - .withConfigProperty(FIELDS_TO_TRANSFORM); - } - } - } + public static class Config extends DLPTransformPluginConfig { } } diff --git a/src/main/java/io/cdap/plugin/dlp/Utils.java b/src/main/java/io/cdap/plugin/dlp/Utils.java index 899df1a..e574e0d 100644 --- a/src/main/java/io/cdap/plugin/dlp/Utils.java +++ b/src/main/java/io/cdap/plugin/dlp/Utils.java @@ -16,7 +16,9 @@ package io.cdap.plugin.dlp; +import com.google.common.base.Strings; import com.google.privacy.dlp.v2.FieldId; +import com.google.privacy.dlp.v2.RecordTransformations; import com.google.privacy.dlp.v2.Table; import com.google.privacy.dlp.v2.Value; import com.google.protobuf.Timestamp; @@ -24,6 +26,9 @@ import com.google.type.TimeOfDay; import io.cdap.cdap.api.data.format.StructuredRecord; import io.cdap.cdap.api.data.schema.Schema; +import io.cdap.cdap.etl.api.lineage.field.FieldOperation; +import io.cdap.cdap.etl.api.lineage.field.FieldTransformOperation; +import io.cdap.plugin.dlp.configs.DlpFieldTransformationConfig; import java.time.Instant; import java.time.LocalDate; @@ -32,7 +37,11 @@ import java.time.ZoneId; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Set; import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -280,4 +289,109 @@ public static Table getTableFromStructuredRecord(StructuredRecord record, @Nulla tableBuiler.addRows(rowBuilder.build()); return tableBuiler.build(); } + + /** + * Converts the internal plugin config representation to DLP {@link RecordTransformations} object + * + * @param config DLP transform plugin config + * @return DLP RecordTransformations object that contains the converted values + * @throws Exception if the config cannot be converted into RecordTransformations + */ + protected static RecordTransformations constructRecordTransformationsFromConfig(DLPTransformPluginConfig config) + throws Exception { + RecordTransformations.Builder recordTransformationsBuilder = RecordTransformations.newBuilder(); + List transformationConfigs = config.parseTransformations(); + + recordTransformationsBuilder.addAllFieldTransformations( + transformationConfigs.stream() + .map(DlpFieldTransformationConfig::toFieldTransformation) + .collect(Collectors.toList()) + ); + return recordTransformationsBuilder.build(); + } + + /** + * Constructs the field operations list required for Field Level Lineage in DLP transform plugins (Redact and + * Decrypt) + * + * @param inputSchema The schema of records coming into the transform + * @param config The DLP transform plugin config for the transform + * @return List of {@link FieldOperation} that contains the data for FLL + * @throws Exception if the config cannot be parsed to obtain the list of transformations + */ + protected static List getFieldOperations(Schema inputSchema, DLPTransformPluginConfig config) + throws Exception { + return getFieldOperations(inputSchema, config, ""); + } + + /** + * Constructs the field operations list required for Field Level Lineage in DLP transform plugins (Redact and + * Decrypt) + * + * @param inputSchema The schema of records coming into the transform + * @param config The DLP transform plugin config for the transform + * @param transformNamePrefix String prefix to add to the transform names to differentiate between Redact and Decrypt + * @return List of {@link FieldOperation} that contains the data for FLL + * @throws Exception if the config cannot be parsed to obtain the list of transformations + */ + protected static List getFieldOperations(Schema inputSchema, DLPTransformPluginConfig config, + String transformNamePrefix) throws Exception { + if (!Strings.isNullOrEmpty(transformNamePrefix) && !transformNamePrefix.endsWith(" ")) { + transformNamePrefix += " "; + } + + //Parse config into format 'FieldName': List<>(['transform','filter']) + HashMap> fieldOperationsData = new HashMap<>(); + for (DlpFieldTransformationConfig transformationConfig : config.parseTransformations()) { + for (String field : transformationConfig.getFields()) { + String filterName = String.join(", ", transformationConfig.getFilters()) + .replace("NONE", String.format("Custom Template (%s)", config.templateId)); + + String transformName = transformNamePrefix + transformationConfig.getTransform(); + + if (!fieldOperationsData.containsKey(field)) { + fieldOperationsData.put(field, Collections.singletonList(new String[]{transformName, filterName})); + } else { + fieldOperationsData.get(field).add(new String[]{transformName, filterName}); + } + } + } + + for (Schema.Field field : inputSchema.getFields()) { + if (!fieldOperationsData.containsKey(field.getName())) { + fieldOperationsData.put(field.getName(), Collections.singletonList(new String[]{"Identity", ""})); + } + } + + List fieldOperations = new ArrayList<>(); + for (String fieldName : fieldOperationsData.keySet()) { + StringBuilder descriptionBuilder = new StringBuilder(); + StringBuilder nameBuilder = new StringBuilder(); + descriptionBuilder.append("Applied "); + boolean first = true; + for (String[] transformFilterPair : fieldOperationsData.get(fieldName)) { + + String transformName = transformFilterPair[0]; + String filterNames = transformFilterPair[1]; + + if (first) { + descriptionBuilder.append(" "); + } + descriptionBuilder.append(String.format("'%s' transform on contents ", transformName)); + if (filterNames.length() > 0) { + descriptionBuilder.append(" matching ").append(filterNames); + } + descriptionBuilder.append(",\n"); + nameBuilder.append(transformName).append(" ,"); + first = false; + } + nameBuilder.deleteCharAt(nameBuilder.length() - 1); + descriptionBuilder.delete(descriptionBuilder.length() - 2, descriptionBuilder.length() - 1); + nameBuilder.append("on ").append(fieldName); + fieldOperations + .add(new FieldTransformOperation(nameBuilder.toString(), descriptionBuilder.toString(), + Collections.singletonList(fieldName), fieldName)); + } + return fieldOperations; + } } diff --git a/widgets/SensitiveRecordDecrypt-transform.json b/widgets/SensitiveRecordDecrypt-transform.json new file mode 100644 index 0000000..937691d --- /dev/null +++ b/widgets/SensitiveRecordDecrypt-transform.json @@ -0,0 +1,337 @@ +{ + "metadata": { + "spec-version": "1.0" + }, + "display-name": "Decrypt", + "configuration-groups": [ + { + "label": "Custom Template", + "properties": [ + { + "widget-type": "toggle", + "name": "customTemplateEnabled", + "label": "Use custom template", + "widget-attributes": { + "default": "false", + "on": { + "value": "true", + "label": "Yes" + }, + "off": { + "value": "false", + "label": "No" + } + } + }, + { + "widget-type": "textbox", + "label": "Template ID", + "name": "templateId", + "widget-attributes": { + "placeholder": "Template ID" + } + } + ] + }, + { + "label": "Decrypt", + "properties": [ + { + "widget-type": "dlp", + "label": "Fields to Transform", + "name": "fieldsToTransform", + "widget-attributes": { + "filters": [ + { + "id": "NONE", + "label": "Custom Template" + }, + { + "id": "DEMOGRAPHIC", + "label": "Demographics" + }, + { + "id": "LOCATION", + "label": "Location" + }, + { + "id": "TAX", + "label": "Tax IDs" + }, + { + "id": "CREDIT_CARD", + "label": "Credit Card Numbers" + }, + { + "id": "PASSPORT", + "label": "Passport Numbers" + }, + { + "id": "HEALTH", + "label": "Health IDs" + }, + { + "id": "NATIONAL_ID", + "label": "National Identification" + }, + { + "id": "DRIVER_LICENSE", + "label": "Driver License IDs" + } + ], + "transforms": [ + { + "label": "Format Preserving Encryption", + "name": "FORMAT_PRESERVING_ENCRYPTION", + "supportedTypes": [ + "string" + ], + "options": [ + { + "name": "keyType", + "widget-type": "select", + "label": "Crypto Key Type", + "widget-attributes": { + "description": "Type of key to use for the cryptographic hash.", + "default": "TRANSIENT", + "values": [ + { + "value": "TRANSIENT", + "label": "Transient" + }, + { + "value": "UNWRAPPED", + "label": "Unwrapped Key" + }, + { + "value": "KMS_WRAPPED", + "label": "KMS Wrapped Key" + } + ] + } + }, + { + "name": "name", + "widget-type": "textbox", + "label": "Transient Key Name", + "widget-attributes": { + "macro": "true", + "placeholder": "Transient key name", + "description": "Optional name for transient key that will be generated" + } + }, + { + "name": "key", + "widget-type": "textbox", + "label": "Unwrapped Key", + "widget-attributes": { + "macro": "true", + "placeholder": "Unwrapped key", + "description": "Base64 encoded key to be used for cryptographic hash. Key must be must be 16, 24 or 32 bytes long." + } + }, + { + "name": "wrappedKey", + "widget-type": "textbox", + "label": "Wrapped Key", + "widget-attributes": { + "macro": "true", + "placeholder": "Wrapped key", + "description": "Wrapped key to be unwrapped using KMS key" + } + }, + { + "name": "cryptoKeyName", + "widget-type": "textbox", + "label": "KMS Resource ID", + "widget-attributes": { + "macro": "true", + "placeholder": "projects/.../locations/.../keyRings/.../cryptoKeys/...", + "description": "Resource ID of the key stored in Key Management Service that will be used to unwrap the wrapped key. The key version should be removed from the resource ID, the primary version will be used." + } + }, + { + "name": "surrogateInfoTypeName", + "widget-type": "textbox", + "label": "Surrogate Type Name", + "widget-attributes": { + "macro": "true", + "placeholder": "Surrogate Type Name", + "required": "true", + "description": "The custom infoType name to annotate the surrogate with. This annotation will be applied to the surrogate by prefixing it with the name of the custom infoType followed by the number of characters comprising the surrogate." + } + }, + { + "name": "alphabet", + "widget-type": "select", + "label": "Alphabet Type", + "widget-attributes": { + "description": "Type of alphabet that the value to be encrypted uses.", + "default": "ALPHA_NUMERIC", + "required": "true", + "values": [ + { + "value": "ALPHA_NUMERIC", + "label": "Alpha-Numeric" + }, + { + "value": "HEXADECIMAL", + "label": "Hexadecimal" + }, + { + "value": "NUMERIC", + "label": "Numeric" + }, + { + "value": "UPPER_CASE_ALPHA_NUMERIC", + "label": "Uppercase Alpha-Numeric" + }, + { + "value": "CUSTOM", + "label": "Custom" + } + ] + } + }, + { + "name": "customAlphabet", + "widget-type": "textbox", + "label": "Custom Alphabet Type", + "widget-attributes": { + "macro": "true", + "placeholder": "ABC...", + "description": "Set of ASCII characters used in the value to be encrypted. The alphabet can be between 2 and 95 characters long, order of characters does not matter. Ex. for telephone numbers the alphabet might be '()-012346789'." + } + }, + { + "widget-type": "input-field-selector", + "label": "Context", + "name": "context", + "widget-attributes": { + "allowedTypes": [ + "string" + ], + "description": "(Optional) Provide additional field to be used as Context. If the primary value is the same but the Context field value is different then two different encrypted values will be generated." + } + } + ], + "filters": [ + { + "name": "custom alphabet", + "condition": { + "property": "alphabet", + "operator": "equal to", + "value": "CUSTOM" + }, + "show": [ + { + "name": "customAlphabet" + } + ] + }, + { + "name": "transient key rules", + "condition": { + "property": "keyType", + "operator": "equal to", + "value": "TRANSIENT" + }, + "show": [ + { + "name": "name" + } + ] + }, + { + "name": "unwrapped key rules", + "condition": { + "property": "keyType", + "operator": "equal to", + "value": "UNWRAPPED" + }, + "show": [ + { + "name": "key" + } + ] + }, + { + "name": "kms wrapped key rules", + "condition": { + "property": "keyType", + "operator": "equal to", + "value": "KMS_WRAPPED" + }, + "show": [ + { + "name": "wrappedKey" + }, + { + "name": "cryptoKeyName" + } + ] + } + ] + } + ] + } + } + ] + }, + { + "label": "Credentials", + "properties": [ + { + "widget-type": "textbox", + "label": "Service Account Path", + "name": "serviceFilePath", + "widget-attributes": { + "default": "auto-detect" + } + }, + { + "widget-type": "textbox", + "label": "Project Id", + "name": "project", + "widget-attributes": { + "default": "auto-detect" + } + } + ] + } + ], + "outputs": [ + { + "name": "schema", + "widget-type": "schema", + "widget-attributes": { + "schema-types": [ + "boolean", + "int", + "long", + "float", + "double", + "bytes", + "string", + "map" + ], + "schema-default-type": "string" + } + } + ], + "filters": [ + { + "name": "Filter for simple condition objects - 1", + "condition": { + "property": "customTemplateEnabled", + "operator": "equal to", + "value": "true" + }, + "show": [ + { + "name": "templateId" + } + ] + } + ] +} diff --git a/widgets/SensitiveRecordRedaction-transform.json b/widgets/SensitiveRecordRedaction-transform.json index 0cbae68..154eb4e 100644 --- a/widgets/SensitiveRecordRedaction-transform.json +++ b/widgets/SensitiveRecordRedaction-transform.json @@ -335,9 +335,10 @@ } } ] + }, { - "label": "Format-Preserving Encryption", + "label": "Format Preserving Encryption", "name": "FORMAT_PRESERVING_ENCRYPTION", "supportedTypes": [ "string"