diff --git a/README.md b/README.md index e5b8977..2f4312a 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ # refs -Compile YAML, JSON or INI config files together through file path references using `$ref` setting +Compile and merge YAML, JSON or INI config files together through file path references ### Install: diff --git a/lib/processor-ini.js b/lib/processor-ini.js index 46ef308..6099fb6 100644 --- a/lib/processor-ini.js +++ b/lib/processor-ini.js @@ -1,50 +1,19 @@ 'use strict'; const Promise = require('bluebird'); -const path = require('path'); const fs = require('fs'); const ini = require('ini'); +const extract = require('./utils/extract'); +const transform = require('./utils/transform'); function process(filePath, key) { - return new Promise((resolve, reject) => { - let fileData = ''; - let data = {}; - try { - fileData = fs.readFileSync(filePath, 'utf-8'); - data = ini.parse(fileData); - } catch (err) { - let returnErr = err; - if (fileData === '') { - returnErr = 'Empty file, nothing to process.'; + return extract(filePath, ini.parse) + .then((dataString) => { + if (dataString.indexOf('$merge') !== -1) { + return Promise.reject('INI config does not support $merge settings.'); } - return reject(returnErr); - } - let dataString = JSON.stringify(data); - const refMatches = dataString.match(/{"\$ref":"(.*?)"}/g); - const baseDir = path.dirname(filePath); - if (refMatches && refMatches.length > 0) { - const iniFileList = []; - refMatches.forEach((matchKey) => { - const iniFile = matchKey.match(/{"\$ref":"(.*)"}/)[1]; - const refFilePath = path.resolve(`${baseDir}/${iniFile}`); - iniFileList.push(process(refFilePath, matchKey)); - }); - return Promise.all(iniFileList) - .then((results) => { - results.forEach((result) => { - dataString = dataString.replace(result.key, result.dataString); - }); - resolve({ - dataString, - key, - }); - }); - } - return resolve({ - dataString, - key, + return transform(dataString, key, filePath, process); }); - }); } function write(outputFile, compiled) { diff --git a/lib/processor-json.js b/lib/processor-json.js index d5cfa62..856680c 100644 --- a/lib/processor-json.js +++ b/lib/processor-json.js @@ -1,49 +1,14 @@ 'use strict'; const Promise = require('bluebird'); -const path = require('path'); +// const path = require('path'); const fs = require('fs'); +const extract = require('./utils/extract'); +const transform = require('./utils/transform'); function process(filePath, key) { - return new Promise((resolve, reject) => { - let dataString = ''; - try { - dataString = fs.readFileSync(filePath, 'utf-8'); - const parsedData = JSON.parse(dataString); - dataString = JSON.stringify(parsedData); - } catch (err) { - let returnErr = err; - if (dataString === '') { - returnErr = 'Empty file, nothing to process.'; - } - return reject(returnErr); - } - - const refMatches = dataString.match(/{"\$ref":"(.*?)"}/g); - const baseDir = path.dirname(filePath); - if (refMatches && refMatches.length > 0) { - const jsonFileList = []; - refMatches.forEach((matchKey) => { - const jsonFile = matchKey.match(/{"\$ref":"(.*)"}/)[1]; - const refFilePath = path.resolve(`${baseDir}/${jsonFile}`); - jsonFileList.push(process(refFilePath, matchKey)); - }); - return Promise.all(jsonFileList) - .then((results) => { - results.forEach((result) => { - dataString = dataString.replace(result.key, result.dataString); - }); - resolve({ - dataString, - key, - }); - }); - } - return resolve({ - dataString, - key, - }); - }); + return extract(filePath, JSON.parse) + .then(dataString => transform(dataString, key, filePath, process)); } function write(outputFile, compiled) { diff --git a/lib/processor-yaml.js b/lib/processor-yaml.js index 935e6c8..e410c71 100644 --- a/lib/processor-yaml.js +++ b/lib/processor-yaml.js @@ -2,50 +2,22 @@ const yaml = require('node-yaml'); const Promise = require('bluebird'); -const path = require('path'); +// const path = require('path'); +const extract = require('./utils/extract'); +const transform = require('./utils/transform'); const options = { encoding: 'utf8', schema: yaml.schema.defaultFull, }; +function parser(data) { + return yaml.parse(data, options); +} + function process(filePath, key) { - return new Promise((resolve, reject) => { - yaml.read(filePath, options, (err, data) => { - let returnErr = err; - if (!err && data === undefined) { - returnErr = 'Empty file, nothing to process.'; - } - if (returnErr) { - return reject(returnErr); - } - let dataString = JSON.stringify(data); - const refMatches = dataString.match(/{"\$ref":"(.*?)"}/g); - const baseDir = path.dirname(filePath); - if (refMatches && refMatches.length > 0) { - const yamlFileList = []; - refMatches.forEach((matchKey) => { - const yamlFile = matchKey.match(/{"\$ref":"(.*)"}/)[1]; - const refFilePath = path.resolve(`${baseDir}/${yamlFile}`); - yamlFileList.push(process(refFilePath, matchKey)); - }); - return Promise.all(yamlFileList) - .then((results) => { - results.forEach((result) => { - dataString = dataString.replace(result.key, result.dataString); - }); - resolve({ - dataString, - key, - }); - }); - } - return resolve({ - dataString, - key, - }); - }); - }); + return extract(filePath, parser) + .then(dataString => transform(dataString, key, filePath, process)); } function write(outputFile, compiled) { diff --git a/lib/utils/extract.js b/lib/utils/extract.js new file mode 100644 index 0000000..9a5b7eb --- /dev/null +++ b/lib/utils/extract.js @@ -0,0 +1,27 @@ +'use strict'; + +const Promise = require('bluebird'); +const fs = require('fs'); + + +function extract(filePath, parser) { + return new Promise((resolve, reject) => { + if (!filePath || filePath === undefined) { + return reject('Requires a file path to process.'); + } + let fileData = ''; + let data = {}; + try { + fileData = fs.readFileSync(filePath, 'utf-8'); + if (fileData === '') { + throw new Error('Empty file, nothing to process.'); + } + data = parser(fileData); + } catch (err) { + return reject(err.message); + } + return resolve(JSON.stringify(data)); + }); +} + +module.exports = extract; diff --git a/lib/utils/transform.js b/lib/utils/transform.js new file mode 100644 index 0000000..653cfbb --- /dev/null +++ b/lib/utils/transform.js @@ -0,0 +1,66 @@ +'use strict'; + +const Promise = require('bluebird'); +const path = require('path'); + +function transform(dataStr, key, filePath, process) { + return new Promise((resolve, reject) => { + let dataString = dataStr; + const refMatches = dataString.match(/{"\$ref":"(.*?)"}/g); + const baseDir = path.dirname(filePath); + if (refMatches && refMatches.length > 0) { + const refFileList = []; + refMatches.forEach((matchKey) => { + const refFile = matchKey.match(/{"\$ref":"(.*)"}/)[1]; + const refFilePath = path.resolve(`${baseDir}/${refFile}`); + refFileList.push(process(refFilePath, matchKey)); + }); + return Promise.all(refFileList) + .then((results) => { + const mergeMatches = dataString.match(/{"\$merge":\[(.*?)\]}/g); + if (mergeMatches && mergeMatches.length > 0) { + mergeMatches.forEach((mergeMatchKey) => { + const localMatches = []; + const localRefMatches = mergeMatchKey.match(/{"\$ref":"(.*?)"}/g); + if (!localRefMatches || localRefMatches.length === 0) { + reject('Malformed merge setting, please check the input file.'); + return; + } + localRefMatches.forEach((refMatchKey) => { + const j = results.length; + for (let i = 0; i < j; i += 1) { + const result = results[i]; + if (result.key === refMatchKey) { + const end = (result.dataString.length - 2); + localMatches.push(result.dataString.substr(1, end)); + break; + } + } + }); + const localMatchesStr = `{${localMatches.join(',')}}`; + dataString = dataString.replace(mergeMatchKey, localMatchesStr); + return; + }); + } else { + results.forEach((result) => { + dataString = dataString.replace(result.key, result.dataString); + }); + } + return resolve({ + dataString, + key, + }); + }); + } + const mergeMatches = dataString.match(/{"\$merge":\[(.*?)\]}/g); + if (mergeMatches && mergeMatches.length > 0) { + return reject('Malformed merge setting, please check the input file.'); + } + return resolve({ + dataString, + key, + }); + }); +} + +module.exports = transform; diff --git a/package.json b/package.json index a6d3d0d..986110d 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "refs", - "version": "0.9.0", - "description": "Compile YAML, JSON or INI config files together through file path references using `$ref` setting", + "version": "0.9.1", + "description": "Compile and merge YAML, JSON or INI config files together through file path references", "main": "index.js", "preferGlobal": true, "bin": { diff --git a/test/data/file-merge.ini b/test/data/file-merge.ini new file mode 100644 index 0000000..eb5f5ab --- /dev/null +++ b/test/data/file-merge.ini @@ -0,0 +1,3 @@ +[test] +$merge[]=$ref:./file.yaml +$merge[]=$ref:./file-refs.yaml diff --git a/test/data/file-merge.yaml b/test/data/file-merge.yaml new file mode 100644 index 0000000..0e8e5a9 --- /dev/null +++ b/test/data/file-merge.yaml @@ -0,0 +1,4 @@ +test: + $merge: + - $ref: ./file.yaml + - $ref: ./file-refs.yaml diff --git a/test/test-lib-ini-processor.js b/test/test-lib-ini-processor.js index ab6d237..93c5fb9 100644 --- a/test/test-lib-ini-processor.js +++ b/test/test-lib-ini-processor.js @@ -8,6 +8,7 @@ const td = require('testdouble'); describe('INI Tests', () => { const INI_FILE = '/tmp/file.ini'; const INI_REF_FILE = '/tmp/file-refs.ini'; + const INI_MERGE_FILE = '/tmp/file-merge.ini'; const INI_FILE_WRITE = '/tmp/file-write.ini'; beforeEach(() => {}); @@ -24,6 +25,11 @@ describe('INI Tests', () => { } catch (e) { // suppress error } + try { + fs.unlinkSync(INI_MERGE_FILE); + } catch (e) { + // suppress error + } try { fs.unlinkSync(INI_FILE_WRITE); } catch (e) { @@ -39,7 +45,7 @@ describe('INI Tests', () => { done('Rejection failed.'); }) .catch((err) => { - should(err).be.eql('Empty file, nothing to process.'); + should(err).be.eql('Requires a file path to process.'); done(); }); }); @@ -61,7 +67,7 @@ describe('INI Tests', () => { }); }); - it('process: should process the file with no refs', (done) => { + it('process: should process the file with no ref settings', (done) => { const iniContent = fs.readFileSync(INI_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); fs.writeFileSync(INI_FILE, iniContent, 'utf-8'); const iniProcessor = require('../lib/processor-ini'); @@ -79,7 +85,7 @@ describe('INI Tests', () => { }); }); - it('process: should process the file with refs', (done) => { + it('process: should process the file with ref settings', (done) => { const iniContent = fs.readFileSync(INI_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); const iniRefContent = fs.readFileSync(INI_REF_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); fs.writeFileSync(INI_FILE, iniContent, 'utf-8'); @@ -99,6 +105,25 @@ describe('INI Tests', () => { }); }); + it('process: should throw error with merge settings', (done) => { + const iniContent = fs.readFileSync(INI_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + const iniRefContent = fs.readFileSync(INI_REF_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + const iniMergeContent = fs.readFileSync(INI_MERGE_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + fs.writeFileSync(INI_FILE, iniContent, 'utf-8'); + fs.writeFileSync(INI_REF_FILE, iniRefContent, 'utf-8'); + fs.writeFileSync(INI_MERGE_FILE, iniMergeContent, 'utf-8'); + const iniProcessor = require('../lib/processor-ini'); + + iniProcessor.process(INI_MERGE_FILE) + .then(() => { + done('Rejection failed.'); + }) + .catch((err) => { + should(err).be.eql('INI config does not support $merge settings.'); + done(); + }); + }); + it('write: should throw error on write', (done) => { td.replace(fs, 'writeFile', (outputFile, compiled, options, cb) => cb('An error occurred.')); const iniProcessor = require('../lib/processor-ini'); @@ -146,12 +171,6 @@ describe('INI Tests', () => { }); it('dump: should dump to output file', (done) => { - // td.replace('node-ini', { - // dump: compiled => JSON.stringify(compiled), - // schema: { - // defaultFull: ini.schema.defaultFull, - // }, - // }); const iniProcessor = require('../lib/processor-ini'); iniProcessor.dump({ test: true }) diff --git a/test/test-lib-json-processor.js b/test/test-lib-json-processor.js index 11fbb55..5b53472 100644 --- a/test/test-lib-json-processor.js +++ b/test/test-lib-json-processor.js @@ -7,6 +7,7 @@ const td = require('testdouble'); describe('JSON Tests', () => { const JSON_FILE = '/tmp/file.json'; const JSON_REF_FILE = '/tmp/file-refs.json'; + const JSON_MERGE_FILE = '/tmp/file-merge.json'; const JSON_FILE_WRITE = '/tmp/file-write.json'; beforeEach(() => {}); @@ -23,6 +24,11 @@ describe('JSON Tests', () => { } catch (e) { // suppress error } + try { + fs.unlinkSync(JSON_MERGE_FILE); + } catch (e) { + // suppress error + } try { fs.unlinkSync(JSON_FILE_WRITE); } catch (e) { @@ -38,7 +44,7 @@ describe('JSON Tests', () => { done('Rejection failed.'); }) .catch((err) => { - should(err).be.eql('Empty file, nothing to process.'); + should(err).be.eql('Requires a file path to process.'); done(); }); }); @@ -57,7 +63,7 @@ describe('JSON Tests', () => { }); }); - it('process: should process the file with no refs', (done) => { + it('process: should process the file with no ref settings', (done) => { const jsonContent = fs.readFileSync(JSON_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); fs.writeFileSync(JSON_FILE, jsonContent, 'utf-8'); const jsonProcessor = require('../lib/processor-json'); @@ -75,7 +81,7 @@ describe('JSON Tests', () => { }); }); - it('process: should process the file with refs', (done) => { + it('process: should process the file with ref settings', (done) => { const jsonContent = fs.readFileSync(JSON_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); const jsonRefContent = fs.readFileSync(JSON_REF_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); fs.writeFileSync(JSON_FILE, jsonContent, 'utf-8'); diff --git a/test/test-lib-yaml-processor.js b/test/test-lib-yaml-processor.js index 731a80c..cd0672c 100644 --- a/test/test-lib-yaml-processor.js +++ b/test/test-lib-yaml-processor.js @@ -8,6 +8,7 @@ const td = require('testdouble'); describe('YAML Tests', () => { const YAML_FILE = '/tmp/file.yaml'; const YAML_REF_FILE = '/tmp/file-refs.yaml'; + const YAML_MERGE_FILE = '/tmp/file-merge.yaml'; const YAML_FILE_WRITE = '/tmp/file-write.yaml'; beforeEach(() => {}); @@ -24,6 +25,11 @@ describe('YAML Tests', () => { } catch (e) { // suppress error } + try { + fs.unlinkSync(YAML_MERGE_FILE); + } catch (e) { + // suppress error + } try { fs.unlinkSync(YAML_FILE_WRITE); } catch (e) { @@ -39,7 +45,7 @@ describe('YAML Tests', () => { done('Rejection failed.'); }) .catch((err) => { - should(err.message).be.eql('Path must be a string. Received undefined'); + should(err).be.eql('Requires a file path to process.'); done(); }); }); @@ -58,7 +64,7 @@ describe('YAML Tests', () => { }); }); - it('process: should process the file with no refs', (done) => { + it('process: should process the file with no ref settings', (done) => { const yamlContent = fs.readFileSync(YAML_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); fs.writeFileSync(YAML_FILE, yamlContent, 'utf-8'); const yamlProcessor = require('../lib/processor-yaml'); @@ -76,7 +82,7 @@ describe('YAML Tests', () => { }); }); - it('process: should process the file with refs', (done) => { + it('process: should process the file with ref settings', (done) => { const yamlContent = fs.readFileSync(YAML_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); const yamlRefContent = fs.readFileSync(YAML_REF_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); fs.writeFileSync(YAML_FILE, yamlContent, 'utf-8'); @@ -96,6 +102,61 @@ describe('YAML Tests', () => { }); }); + it('process: should throw error the file with malformed merge settings', (done) => { + fs.writeFileSync(YAML_MERGE_FILE, '$merge:\n - one: true\n - two: true\n', 'utf-8'); + const yamlProcessor = require('../lib/processor-yaml'); + + yamlProcessor.process(YAML_MERGE_FILE) + .then(() => { + done('Rejection failed.'); + }) + .catch((err) => { + should(err).be.eql('Malformed merge setting, please check the input file.'); + done(); + }); + }); + + it('process: should throw error the file with malformed merge settings with ref settings', (done) => { + const yamlContent = fs.readFileSync(YAML_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + const yamlRefContent = fs.readFileSync(YAML_REF_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + const yamlMergeContent = fs.readFileSync(YAML_MERGE_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + fs.writeFileSync(YAML_FILE, yamlContent, 'utf-8'); + fs.writeFileSync(YAML_REF_FILE, yamlRefContent, 'utf-8'); + fs.writeFileSync(YAML_MERGE_FILE, `${yamlMergeContent}\nanother:\n $merge:\n - one: true\n - two: true\n`, 'utf-8'); + const yamlProcessor = require('../lib/processor-yaml'); + + yamlProcessor.process(YAML_MERGE_FILE) + .then(() => { + done('Rejection failed.'); + }) + .catch((err) => { + should(err).be.eql('Malformed merge setting, please check the input file.'); + done(); + }); + }); + + it('process: should process the file with merge settings', (done) => { + const yamlContent = fs.readFileSync(YAML_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + const yamlRefContent = fs.readFileSync(YAML_REF_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + const yamlMergeContent = fs.readFileSync(YAML_MERGE_FILE.replace('/tmp', `${__dirname}/data`), 'utf-8'); + fs.writeFileSync(YAML_FILE, yamlContent, 'utf-8'); + fs.writeFileSync(YAML_REF_FILE, yamlRefContent, 'utf-8'); + fs.writeFileSync(YAML_MERGE_FILE, yamlMergeContent, 'utf-8'); + const yamlProcessor = require('../lib/processor-yaml'); + + yamlProcessor.process(YAML_MERGE_FILE) + .then((results) => { + should(results).be.eql({ + dataString: '{"test":{"test":true,"another":{"test":true}}}', + key: undefined, + }); + done(); + }) + .catch((err) => { + done(err); + }); + }); + it('write: should throw error on write', (done) => { td.replace('node-yaml', { write: (outputFile, compiled, options, cb) => cb('An error occurred.'),