From 1484f94b9b8f0606a75d5be23b0b223de24c06c6 Mon Sep 17 00:00:00 2001 From: Brian Leonard Date: Thu, 22 Feb 2024 10:58:10 -0500 Subject: [PATCH 1/7] update schema model version --- src/setup/loadSchema.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/setup/loadSchema.ts b/src/setup/loadSchema.ts index 58740dd..c774174 100644 --- a/src/setup/loadSchema.ts +++ b/src/setup/loadSchema.ts @@ -1,6 +1,6 @@ import { Schema } from '../types/schema.ts' import { objectPathHandler } from '../utils/objectPathHandler.ts' -import * as schemaDefault from 'https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/latest/schema.json' with { type: 'json' } +import * as schemaDefault from 'https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/1.1.0/schema.json' with { type: 'json' } /** * Load the schema from the specification @@ -12,11 +12,11 @@ export async function loadSchema(version = 'latest'): Promise { let schemaUrl = version const psychdsSchema = typeof Deno !== 'undefined' ? Deno.env.get('psychDS_SCHEMA') : undefined - const schemaOrgUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/external_schemas/schemaorg/schemaorg.json?v=${Date.now()}` + const schemaOrgUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/external_schemas/schemaorg/schemaorg.json?v=34${Date.now()}` if (psychdsSchema !== undefined) { schemaUrl = psychdsSchema } else if (version === 'latest' || versionRegex.test(version)) { - schemaUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/${version}/schema.json?v=${Date.now()}` + schemaUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/${version}/schema.json?v=34${Date.now()}` } try { let schemaModule = await import(schemaUrl, { @@ -26,10 +26,8 @@ export async function loadSchema(version = 'latest'): Promise { const schemaOrgModule = await import(schemaOrgUrl, { with: { type: 'json'}, }) - //console.log(schemaModule) schemaModule.default = {...schemaModule.default, schemaOrg:schemaOrgModule} - //console.log(schemaModule.default) return new Proxy( schemaModule.default as object, objectPathHandler, From fbd60bb2d2a0784b970a18d7fce8b6b9109a248e Mon Sep 17 00:00:00 2001 From: Brian Leonard Date: Thu, 22 Feb 2024 10:59:39 -0500 Subject: [PATCH 2/7] modify evalMap to recognize that 'columns' prop is now 'columnsMatchMetadata' --- src/schema/applyRules.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/schema/applyRules.ts b/src/schema/applyRules.ts index 1639818..67103c6 100644 --- a/src/schema/applyRules.ts +++ b/src/schema/applyRules.ts @@ -94,7 +94,7 @@ import { psychDSFile } from '../types/file.ts'; schemaPath: string, ) => boolean | void | Promise > = { - columns: evalColumns, + columnsMatchMetadata: evalColumns, fields: evalJsonCheck, } From 58deda71ccaceff6b962ec1d8f205bcdd55e411f Mon Sep 17 00:00:00 2001 From: Brian Leonard Date: Thu, 22 Feb 2024 11:01:56 -0500 Subject: [PATCH 3/7] modify unit tests to reflect that all content-based rules for metadata are now located under compiled_metadata rather than dataset_description, and the rules are triggered by encountered a datafile rather than a metadata object (since the rules must be applied to the fully compiled metadata object, whose inheritance depends on datafile location) --- src/schema/applyRules.test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/schema/applyRules.test.ts b/src/schema/applyRules.test.ts index 8039a18..6d79560 100644 --- a/src/schema/applyRules.test.ts +++ b/src/schema/applyRules.test.ts @@ -86,7 +86,7 @@ Deno.test({ dsContext = new psychDSContextDataset({datasetPath:PATH} as ValidatorOptions, ddFile, description) } - const fileName = '/dataset_description.json' + const fileName = '/data/raw_data/study-bfi_data.csv' const file = new psychDSFileDeno(PATH, fileName, ignore) const context = new psychDSContext(fileTree, file, issues,dsContext) await context.asyncLoads() @@ -105,7 +105,7 @@ Deno.test({ const description = await ddFile.text().then((text: string) => JSON.parse(text)) dsContext = new psychDSContextDataset({datasetPath:invPATH} as ValidatorOptions, ddFile, description) } - const fileName = 'dataset_description.json' + const fileName = '/data/raw_data/study-bfi_data.csv' const file = new psychDSFileDeno(invPATH, fileName, ignore) const context = new psychDSContext(invFileTree, file, issues,dsContext) await context.asyncLoads() @@ -123,7 +123,7 @@ Deno.test({ const description = await ddFile.text().then((text: string) => JSON.parse(text)) dsContext = new psychDSContextDataset({datasetPath:noCtxPATH} as ValidatorOptions, ddFile, description) } - const fileName = 'dataset_description.json' + const fileName = '/data/raw_data/study-bfi_data.csv' const file = new psychDSFileDeno(noCtxPATH, fileName, ignore) const context = new psychDSContext(noCtxFileTree, file, issues,dsContext) From 55fce768d66d8b003b9a50ca5adfd762e8a459bd Mon Sep 17 00:00:00 2001 From: Brian Leonard Date: Thu, 22 Feb 2024 11:02:40 -0500 Subject: [PATCH 4/7] misc. updates to ignore list --- src/files/ignore.test.ts | 1 - src/files/ignore.ts | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/files/ignore.test.ts b/src/files/ignore.test.ts index 1ec26f4..fb4a1f5 100644 --- a/src/files/ignore.test.ts +++ b/src/files/ignore.test.ts @@ -7,7 +7,6 @@ Deno.test('Deno implementation of FileIgnoreRules', async (t) => { '/sub-01/anat/sub-01_T1w.nii.gz', '/dataset_description.json', '/README', - '/CHANGES', '/participants.tsv', '/.git/HEAD', '/sub-01/anat/non-bidsy-file.xyz', diff --git a/src/files/ignore.ts b/src/files/ignore.ts index 6b3bac9..66086c3 100644 --- a/src/files/ignore.ts +++ b/src/files/ignore.ts @@ -13,6 +13,7 @@ export async function readPsychDSIgnore(file: psychDSFile) { const defaultIgnores = [ '.git**', + '*.DS_Store', '.datalad/', '.reproman/', 'sourcedata/', @@ -21,11 +22,9 @@ const defaultIgnores = [ 'materials/', 'results/', 'products/', + 'analysis/', 'documentation/', - 'CHANGES*', - 'log/', - '**/meg/*.ds/**', - '**/micr/*.zarr/**', + 'log/' ] /** From 7e8a61e762a4186d48d0efcbb3fb528720e6dbf0 Mon Sep 17 00:00:00 2001 From: Brian Leonard Date: Thu, 22 Feb 2024 11:03:43 -0500 Subject: [PATCH 5/7] modify GenericRule interface to reflect property name change from 'columns' to 'columnsMatchMetadata' --- src/types/schema.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/types/schema.ts b/src/types/schema.ts index e29856e..7c97bba 100644 --- a/src/types/schema.ts +++ b/src/types/schema.ts @@ -68,7 +68,7 @@ export interface GenericRuleOrg { export interface GenericRule { selectors?: string[] checks?: string[] - columns?: Record + columnsMatchMetadata?: boolean additional_columns?: string initial_columns?: string[] fields: Record From 4e00e2805b42304129b669d7ab50664dcc639fb1 Mon Sep 17 00:00:00 2001 From: Brian Leonard Date: Thu, 22 Feb 2024 11:10:57 -0500 Subject: [PATCH 6/7] modify _findRuleMatches to be less cluttered. path property becomes deprecated in cases where the object is not a directory; stem and suffix are mutually exclusive, arbitraryNesting applies to all filename rules --- src/validators/filenameIdentify.ts | 76 ++++++++++++++++++++++-------- 1 file changed, 56 insertions(+), 20 deletions(-) diff --git a/src/validators/filenameIdentify.ts b/src/validators/filenameIdentify.ts index 211799c..68b4494 100644 --- a/src/validators/filenameIdentify.ts +++ b/src/validators/filenameIdentify.ts @@ -58,15 +58,21 @@ export function findFileRules(schema,rulesRecord) { export function _findFileRules(node, path,rulesRecord) { if ( - ('path' in node) || - ('stem' in node) || - (('baseDir' in node) && + ('baseDir' in node) && ('extensions' in node) && - ('suffix')) + (('suffix' in node) || ('stem' in node)) ) { rulesRecord[path] = false return } + //recognize that some objects required or recommended by the spec are directories + if ( + 'path' in node && + 'directory' in node + ){ + rulesRecord[path] = false + return + } else { Object.keys(node).map((key) => { if( @@ -88,8 +94,11 @@ function findRuleMatches(schema, context) { context.filenameRules.length === 0 && context.file.path !== '/.bidsignore' ) { + //if no rules are found to match given file/directory, add NotIncluded warning to indicate + //that the file/directory is not part of the PsychDS specification context.issues.addSchemaIssue('NotIncluded', [context.file]) if(context.file.name === "dataset_description.json"){ + //if global metadata file is located outside of root directory, issue specific warning context.issues.addSchemaIssue( "WrongMetadataLocation", [context.file], @@ -101,28 +110,55 @@ function findRuleMatches(schema, context) { return Promise.resolve() } +function checkFileRules(arbitraryNesting: boolean, hasSuffix: boolean, node, context){ + let baseDirCond: boolean = null + let suffixStemCond: boolean = null + + //if arbitraryNesting applies, then it is only required that the file is located in the correct base directory, + //with any number of subdirectories intervening + if (arbitraryNesting) + baseDirCond = context.baseDir === node.baseDir + //otherwise, the file must be located directly under the baseDir + else{ + //if the baseDir is root, arbitraryNesting does not apply + if(context.baseDir === "/") + baseDirCond = context.path === `/${context.file.name}` + else + baseDirCond = context.path === `/${node.baseDir}/${context.file.name}` + } + + //if the suffix property is present on a rule, then the file should be identified by its suffix + if (hasSuffix) + suffixStemCond = context.suffix === node.suffix + //otherwise, a file should be identified with its stem + else + suffixStemCond = context.file.name.startsWith(node.stem) + + //files are identified by a combination of their baseDir, their extensions, and either their stem or their suffix + if ( + baseDirCond && + node.extensions.includes(context.extension) && + suffixStemCond + ) + return true + else + return false +} + /* Schema rules specifying valid filenames follow a variety of patterns. - * 'path', 'stem' or 'suffixies' contain the most unique identifying + * 'baseDir', 'extensions', 'stem' or 'suffixies' contain the most unique identifying * information for a rule. We don't know what kind of filename the context is, - * so if one of these three match the respective value in the context lets + * so if one of these match the respective value in the context lets * assume that this schema rule is applicable to this file. */ export function _findRuleMatches(node, path, context) { - //console.log(node) - if ( - ('path' in node && context.file.path === node.path) || - ('stem' in node && context.file.name.startsWith(node.stem)) || - ((('baseDir' in node && 'arbitraryNesting' in node && node.arbitraryNesting && context.baseDir === node.baseDir) || - ('baseDir' in node && 'arbitraryNesting' in node && !node.arbitraryNesting && context.path === `/${node.baseDir}/${context.file.name}`)) && - ('extensions' in node && node.extensions.includes(context.extension)) && - ('suffix' in node && context.suffix === node.suffix)) - ) { - context.filenameRules.push(path) - return + if ('arbitraryNesting' in node){ + if (checkFileRules(node.arbitraryNesting,'suffix' in node, node, context)){ + context.filenameRules.push(path) + return + } } - if ( - !('path' in node || 'stem' in node || 'baseDir' in node || 'extensions' in node || 'suffix' in node) - ) { + else { Object.keys(node).map((key) => { if( typeof node[key] === 'object' From f97afbb43ada86cb9544ddd15dcafbee696d0e85 Mon Sep 17 00:00:00 2001 From: Brian Leonard Date: Thu, 22 Feb 2024 11:11:32 -0500 Subject: [PATCH 7/7] update unit tests to accommodate new functionality --- src/validators/filenameIdentify.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/validators/filenameIdentify.test.ts b/src/validators/filenameIdentify.test.ts index 46e0163..0bbab74 100644 --- a/src/validators/filenameIdentify.test.ts +++ b/src/validators/filenameIdentify.test.ts @@ -21,6 +21,9 @@ const ignore = new FileIgnoreRules([]) const node = { stem: 'dataset_description', + arbitraryNesting: true, + baseDir: "/", + extensions: [".json"] } const recurseNode = {