KEY-684: Export without secrets (#94)

* export options; included/exluded props; changelog

* tests

* Update changelog

* Just use `AUTH0_EXPORT_IDENTIFIERS` in config

instead of `export_ids` as command line option

* return 'export_ids' param, which will override config value

* --export_ids options added to the readme

CHANGELOG.md

@@ -4,6 +4,14 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [3.0.0] - Unreleased
+### Added
+- `INCLUDED_PROPS` option has been added to the config. It allows user to export properties which are excluded by default (like `client_secret`).
+- `EXCLUDED_PROPS` option has been added to the config. It allows user to exclude any unwanted properties from exported objects.
+
+### Changed
+- `--strip` option has been removed from `export` command. Now IDs will be stripped by default, but you can use `AUTH0_EXPORT_IDENTIFIERS: true` to prevent.
+
 ## [2.3.2] - Unreleased
 ### Changed
 - set `enable_sso` and `sandbox_version` as readonly properties
@@ -22,11 +30,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Update environment variable explanation in READMEs. #90
 - Sanitize file and folder names. #92
 
-## [2.2.5] - 2018-02-04
+## [2.2.5] - 2019-02-04
 ### Changed
 - Fix for using the wrong proxy reference. #80
 
-## [2.2.4] - 2018-01-17
+## [2.2.4] - 2019-01-17
 ### Changed
 - Fix various schema validation issues. auth0-extensions/auth0-source-control-extension-tools PRs #52 thru #57
 

README.md

@@ -13,7 +13,7 @@ Supported Features
   - Clients / Applications
   - Resource Servers (APIs)
   - Pages
-  - Email Templates and Provider 
+  - Email Templates and Provider
   - Guardian Settings
 - Configuration options
   - Defined Directory Structure
@@ -132,8 +132,9 @@ import { deploy, dump } from 'auth0-deploy-cli';
 
 const config = {
   AUTH0_DOMAIN: process.env.AUTH0_DOMAIN,
-  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_ID,
-  AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_SECRET,
+  AUTH0_CLIENT_SECRET: process.env.AUTH0_CLIENT_SECRET,
+  AUTH0_CLIENT_ID: process.env.AUTH0_CLIENT_ID,
+  AUTH0_EXPORT_IDENTIFIERS: false,
   AUTH0_ALLOW_DELETE: false
 };
 
@@ -144,7 +145,7 @@ dump({
   base_path: basePath,                          // Allow to override basepath, if not take from input_file
   config_file: configFile,                      // Option to a config json
   config: configObj,                            // Option to sent in json as object
-  strip,                                        // Strip the identifier field for each object type
+  export_ids: exportIds,                        // Export the identifier field for each object type
   secret                                        // Optionally pass in auth0 client secret seperate from config
 })
   .then(() => console.log('yey export was successful'))
@@ -189,8 +190,8 @@ Options:
   --proxy_url, -p  A url for proxying requests, only set this if you are behind a proxy.  [string]
 
 Examples:
-  a0deploy export -c config.json --strip -f yaml -o path/to/export       Dump Auth0 config to folder in YAML format
-  a0deploy export -c config.json --strip -f directory -o path/to/export  Dump Auth0 config to folder in directory format
+  a0deploy export -c config.json -f yaml -o path/to/export               Dump Auth0 config to folder in YAML format
+  a0deploy export -c config.json -f directory -o path/to/export          Dump Auth0 config to folder in directory format
   a0deploy import -c config.json -i tenant.yaml                          Deploy Auth0 via YAML
   a0deploy import -c config.json -i path/to/files                        Deploy Auth0 via Path
 
@@ -221,7 +222,7 @@ The deploy task should follow these steps:
  1.  Update the local repo to the latest. (each environment should have its own copy of the repo set to its own branch)
  1.  If there are changes, call a0deploy
  1.  Run a suite of tests to confirm configuration is working
- 1.  Optional:  merge to next branch
+ 1.  Optional: merge to next branch
 
 ### Use keyword mappings to handle differences between the environments
 You should not have to store differences between environments in the Deploy Configuration Repository.  Use the keyword mappings to allow the repository to be environment agnostic, and instead store the differences in the separate config.json files for each environment that are stored on the CI server.

examples/directory/README.md

@@ -62,9 +62,9 @@ repository =>
 ## Example Export
 You can export your current tenant configuration. For example the following command will export your tenant configuration.
 
-`a0deploy export -c config.json --strip -f directory -o path/to/export`
+`a0deploy export -c config.json -f directory -o path/to/export`
 
-> NOTE: The option --strip is used to remove the identifier fields from the Auth0 objects. This means when importing into another Auth0 Tenant new id's are generated otherwise the import will fail as the tool cannot find the existing objects by their id.
+> NOTE: The config value `AUTH0_EXPORT_IDENTIFIERS: true` (or `--export_ids` option) can be used to export the identifier fields to the Auth0 objects. This means you won't be able to import these objects as the tool cannot find the existing objects by their id.
 
 > NOTE: Some of the settings cannot be exported for example emailProvider credentials, rulesConfigs values and others. After export you may need to update the `tenant.yaml` values if you experience schema errors on import.
 
@@ -107,7 +107,13 @@ Here is the example of a config.json:
   "AUTH0_EXCLUDED_RULES": [
     "rule-1-name",
     "rule-2-name"
-  ]
+  ],
+  "INCLUDED_PROPS": {
+    "clients": [ "client_secret" ]
+  },
+  "EXCLUDED_PROPS": {
+    "connections": [ "options.client_secret" ]
+  }
 }
 ```
 
@@ -116,7 +122,7 @@ The `auth0-deploy-cli` supports environment variables replacements, also known a
 
 Environment variables can be set on the terminal and within the `config.json`. At run time the variables defined in your terminal and `config.json` will be merged. You can disable this via the command line with the `--no-env` option. The terminal variables will take priority over `config.json`
 
-There are two ways to use the keyword mappings in your Auth0 Tenant configuration files. You can inject values using `@@key@@` or `##key##`. 
+There are two ways to use the keyword mappings in your Auth0 Tenant configuration files. You can inject values using `@@key@@` or `##key##`.
 
 If you use the `@` symbols, it will do a JSON.stringify on your value before replacing it. So if it is a string it will add quotes, if it is an array or object it will add braces.
 

examples/directory/config.json.example

@@ -10,5 +10,11 @@
   "AUTH0_EXCLUDED_RULES": [
     "rule-1-name",
     "rule-2-name"
-  ]
+  ],
+  "INCLUDED_PROPS": {
+    "clients": [ "client_secret" ]
+  },
+  "EXCLUDED_PROPS": {
+    "connections": [ "options.client_secret" ]
+  }
 }

examples/yaml/README.md

@@ -10,9 +10,9 @@ For more information on YAML please refer to [http://yaml.org/](http://yaml.org/
 ## Example Export
 You can export your current tenant configuration. For example the following command will export your tenant configuration.
 
-`a0deploy export -c config.json --strip -f yaml -o path/to/export`
+`a0deploy export -c config.json -f yaml -o path/to/export`
 
-> NOTE: The option --strip is used to remove the identifier fields from the Auth0 objects. This means when importing into another Auth0 Tenant new id's are generated otherwise the import will fail as the tool cannot find the existing objects by their id.
+> NOTE: The config value `AUTH0_EXPORT_IDENTIFIERS: true` (or `--export_ids` option) can be used to export the identifier fields to the Auth0 objects. This means you won't be able to import these objects as the tool cannot find the existing objects by their id.
 
 > NOTE: Some of the settings cannot be exported for example emailProvider credentials, rulesConfigs values and others. After export you may need to update the `tenant.yaml` values if you experience schema errors on import.
 
@@ -50,6 +50,12 @@ Here is the example of a config.json:
       "https://somedomain.com"
     ],
     "YOUR_STRING_KEY": "some environment specific string"
+  },
+  "INCLUDED_PROPS": {
+    "clients": [ "client_secret" ]
+  },
+  "EXCLUDED_PROPS": {
+    "connections": [ "options.client_secret" ]
   }
 }
 ```
@@ -59,7 +65,7 @@ The `auth0-deploy-cli` supports environment variables replacements, also known a
 
 Environment variables can be set on the terminal and within the `config.json`. At run time the variables defined in your terminal and `config.json` will be merged. You can disable this via the command line with the `--no-env` option. The terminal variables will take priority over `config.json`
 
-There are two ways to use the keyword mappings in your Auth0 Tenant configuration files. You can inject values using `@@key@@` or `##key##`. 
+There are two ways to use the keyword mappings in your Auth0 Tenant configuration files. You can inject values using `@@key@@` or `##key##`.
 
 If you use the `@` symbols, it will do a JSON.stringify on your value before replacing it. So if it is a string it will add quotes, if it is an array or object it will add braces.
 

examples/yaml/config.json.example

@@ -10,5 +10,11 @@
   "AUTH0_EXCLUDED_RULES": [
     "rule-1-name",
     "rule-2-name"
-  ]
+  ],
+  "INCLUDED_PROPS": {
+    "clients": [ "client_secret" ]
+  },
+  "EXCLUDED_PROPS": {
+    "connections": [ "options.client_secret" ]
+  }
 }

src/args.js

@@ -58,20 +58,20 @@ export default yargs
       describe: 'The JSON configuration file.',
       type: 'string'
     },
-    strip: {
-      alias: 's',
-      describe: 'Strip the identifier field for each object type.',
-      type: 'boolean',
-      default: false
-    },
     secret: {
       alias: 'x',
       describe: 'The client secret, this allows you to encrypt the secret in your build configuration instead of storing it in a config file',
       type: 'string'
+    },
+    export_ids: {
+      alias: 'e',
+      describe: 'Export identifier field for each object type.',
+      type: 'boolean',
+      default: false
     }
   })
-  .example('$0 export -c config.json --strip -f yaml -o path/to/export', 'Dump Auth0 config to folder in YAML format')
-  .example('$0 export -c config.json --strip -f directory -o path/to/export', 'Dump Auth0 config to folder in directory format')
+  .example('$0 export -c config.json -f yaml -o path/to/export', 'Dump Auth0 config to folder in YAML format')
+  .example('$0 export -c config.json -f directory -o path/to/export', 'Dump Auth0 config to folder in directory format')
   .example('$0 import -c config.json -i tenant.yaml', 'Deploy Auth0 via YAML')
   .example('$0 import -c config.json -i path/to/files', 'Deploy Auth0 via Path')
   .epilogue('See README (https://github.com/auth0/auth0-deploy-cli) for more in-depth information on configuration and setup.')

src/commands/export.js

@@ -12,7 +12,7 @@ export default async function deploy(params) {
     base_path: basePath,
     config_file: configFile,
     config: configObj,
-    strip,
+    export_ids: exportIds,
     secret
   } = params;
 
@@ -26,7 +26,6 @@ export default async function deploy(params) {
     AUTH0_INPUT_FILE: outputFolder,
     AUTH0_BASE_PATH: basePath,
     AUTH0_CONFIG_FILE: configFile,
-    AUTH0_STRIP_IDENTIFIERS: strip,
     ...configObj || {}
   };
 
@@ -36,6 +35,11 @@ export default async function deploy(params) {
     overrides.AUTH0_CLIENT_SECRET = secret;
   }
 
+  // Allow passed in export_ids to override the configured one
+  if (exportIds) {
+    overrides.AUTH0_EXPORT_IDENTIFIERS = exportIds;
+  }
+
   // Check output folder
   if (!isDirectory(outputFolder)) {
     log.info(`Creating ${outputFolder}`);

src/context/directory/index.js

@@ -57,14 +57,14 @@ export default class {
     this.assets = auth0.assets;
 
     // Clean known read only fields
-    this.assets = cleanAssets(this.assets);
+    this.assets = cleanAssets(this.assets, this.config);
 
     // Copy clients to be used by handlers which require converting client_id to the name
-    // Must copy as the client_id will be stripped if AUTH0_STRIP_IDENTIFIERS is true
+    // Must copy as the client_id will be stripped if AUTH0_EXPORT_IDENTIFIERS is false
     this.assets.clientsOrig = [ ...this.assets.clients ];
 
     // Optionally Strip identifiers
-    if (this.config.AUTH0_STRIP_IDENTIFIERS) {
+    if (!this.config.AUTH0_EXPORT_IDENTIFIERS) {
       this.assets = stripIdentifiers(auth0, this.assets);
     }
 

src/context/yaml/index.js

@@ -99,13 +99,13 @@ export default class {
     }));
 
     // Clean known read only fields
-    let cleaned = cleanAssets(this.assets);
+    let cleaned = cleanAssets(this.assets, this.config);
 
     // Delete exclude as it's not part of the auth0 tenant config
     delete cleaned.exclude;
 
     // Optionally Strip identifiers
-    if (this.config.AUTH0_STRIP_IDENTIFIERS) {
+    if (!this.config.AUTH0_EXPORT_IDENTIFIERS) {
       cleaned = stripIdentifiers(auth0, cleaned);
     }
 

src/readonly.js

@@ -17,6 +17,7 @@ const readOnly = {
     'flags.disable_impersonation'
   ],
   clients: [
+    'client_secret',
     'callback_url_template',
     'signing_keys',
     'global',
@@ -27,16 +28,42 @@ const readOnly = {
   ]
 };
 
+function getExcludedFields(config) {
+  const strippedFields = { ...readOnly };
+
+  let { EXCLUDED_PROPS: excluded, INCLUDED_PROPS: included } = config;
+  if (typeof excluded !== 'object') excluded = {};
+  if (typeof included !== 'object') included = {};
+
+  Object.entries(excluded).forEach(([ name, fields ]) => {
+    // Do not allow same field to be included and excluded at the same time
+    const intersections = fields.filter(field => included[name] && included[name].includes(field));
+    if (intersections.length > 0) {
+      throw new Error(`EXCLUDED_PROPS should NOT have any intersections with INCLUDED_PROPS. Intersections found: ${name}: ${intersections.join(', ')}`);
+    }
+    strippedFields[name] = (strippedFields[name] || []).concat(fields);
+  });
+
+  Object.entries(included).forEach(([ name, fields ]) => {
+    if (strippedFields[name]) {
+      strippedFields[name] = strippedFields[name].filter(field => !fields.includes(field));
+    }
+  });
+
+  return strippedFields;
+}
+
 function deleteKeys(obj, keys) {
   const newObj = { ...obj };
   keys.forEach(k => dotProp.delete(newObj, k));
   return newObj;
 }
 
-export default function cleanAssets(assets) {
+export default function cleanAssets(assets, config) {
   const cleaned = { ...assets };
+  const excludedFields = getExcludedFields(config);
 
-  Object.entries(readOnly).forEach(([ name, fields ]) => {
+  Object.entries(excludedFields).forEach(([ name, fields ]) => {
     const obj = cleaned[name];
     if (!obj) return;