Merge pull request #46 from psych-ds/filename_fixes

Filename fixes

src/files/ignore.test.ts

@@ -7,7 +7,6 @@ Deno.test('Deno implementation of FileIgnoreRules', async (t) => {
       '/sub-01/anat/sub-01_T1w.nii.gz',
       '/dataset_description.json',
       '/README',
-      '/CHANGES',
       '/participants.tsv',
       '/.git/HEAD',
       '/sub-01/anat/non-bidsy-file.xyz',

src/files/ignore.ts

@@ -13,6 +13,7 @@ export async function readPsychDSIgnore(file: psychDSFile) {
 
 const defaultIgnores = [
   '.git**',
+  '*.DS_Store',
   '.datalad/',
   '.reproman/',
   'sourcedata/',
@@ -21,11 +22,9 @@ const defaultIgnores = [
   'materials/',
   'results/',
   'products/',
+  'analysis/',
   'documentation/',
-  'CHANGES*',
-  'log/',
-  '**/meg/*.ds/**',
-  '**/micr/*.zarr/**',
+  'log/'
 ]
 
 /**

src/schema/applyRules.test.ts

@@ -86,7 +86,7 @@ Deno.test({
           dsContext = new psychDSContextDataset({datasetPath:PATH} as ValidatorOptions, ddFile, description)
         }
 
-        const fileName = '/dataset_description.json'
+        const fileName = '/data/raw_data/study-bfi_data.csv'
         const file = new psychDSFileDeno(PATH, fileName, ignore)
         const context = new psychDSContext(fileTree, file, issues,dsContext)
         await context.asyncLoads()
@@ -105,7 +105,7 @@ Deno.test({
         const description = await ddFile.text().then((text: string) => JSON.parse(text))
         dsContext = new psychDSContextDataset({datasetPath:invPATH} as ValidatorOptions, ddFile, description)
       }
-      const fileName = 'dataset_description.json'
+      const fileName = '/data/raw_data/study-bfi_data.csv'
       const file = new psychDSFileDeno(invPATH, fileName, ignore)
       const context = new psychDSContext(invFileTree, file, issues,dsContext)
       await context.asyncLoads()
@@ -123,7 +123,7 @@ Deno.test({
         const description = await ddFile.text().then((text: string) => JSON.parse(text))
         dsContext = new psychDSContextDataset({datasetPath:noCtxPATH} as ValidatorOptions, ddFile, description)
       }
-      const fileName = 'dataset_description.json'
+      const fileName = '/data/raw_data/study-bfi_data.csv'
       const file = new psychDSFileDeno(noCtxPATH, fileName, ignore)
       const context = new psychDSContext(noCtxFileTree, file, issues,dsContext)
 

src/schema/applyRules.ts

@@ -94,7 +94,7 @@ import { psychDSFile } from '../types/file.ts';
       schemaPath: string,
     ) => boolean | void | Promise<void>
   > = {
-    columns: evalColumns,
+    columnsMatchMetadata: evalColumns,
     fields: evalJsonCheck,
   }
 

src/setup/loadSchema.ts

@@ -1,6 +1,6 @@
 import { Schema } from '../types/schema.ts'
 import { objectPathHandler } from '../utils/objectPathHandler.ts'
-import * as schemaDefault from 'https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/latest/schema.json' with { type: 'json' }
+import * as schemaDefault from 'https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/1.1.0/schema.json' with { type: 'json' }
 
 /**
  * Load the schema from the specification
@@ -12,11 +12,11 @@ export async function loadSchema(version = 'latest'): Promise<Schema> {
   let schemaUrl = version
   const psychdsSchema =
     typeof Deno !== 'undefined' ? Deno.env.get('psychDS_SCHEMA') : undefined
-  const schemaOrgUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/external_schemas/schemaorg/schemaorg.json?v=${Date.now()}`
+  const schemaOrgUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/external_schemas/schemaorg/schemaorg.json?v=34${Date.now()}`
   if (psychdsSchema !== undefined) {
     schemaUrl = psychdsSchema
   } else if (version === 'latest' || versionRegex.test(version)) {
-    schemaUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/${version}/schema.json?v=${Date.now()}`
+    schemaUrl = `https://raw.githubusercontent.com/psych-ds/psych-DS/develop/schema_model/versions/jsons/${version}/schema.json?v=34${Date.now()}`
   }
   try {
     let schemaModule = await import(schemaUrl, {
@@ -26,10 +26,8 @@ export async function loadSchema(version = 'latest'): Promise<Schema> {
     const schemaOrgModule = await import(schemaOrgUrl, {
       with: { type: 'json'},
     })
-    //console.log(schemaModule)
     schemaModule.default = {...schemaModule.default,
       schemaOrg:schemaOrgModule}
-      //console.log(schemaModule.default)
     return new Proxy(
       schemaModule.default as object,
       objectPathHandler,

src/types/schema.ts

@@ -68,7 +68,7 @@ export interface GenericRuleOrg {
 export interface GenericRule {
   selectors?: string[]
   checks?: string[]
-  columns?: Record<string, string>
+  columnsMatchMetadata?: boolean
   additional_columns?: string
   initial_columns?: string[]
   fields: Record<string, SchemaFields>

src/validators/filenameIdentify.test.ts

@@ -21,6 +21,9 @@ const ignore = new FileIgnoreRules([])
 
 const node = {
   stem: 'dataset_description',
+  arbitraryNesting: true,
+  baseDir: "/",
+  extensions: [".json"]
 }
 
 const recurseNode = {

src/validators/filenameIdentify.ts

@@ -58,15 +58,21 @@ export function findFileRules(schema,rulesRecord) {
 
 export function _findFileRules(node, path,rulesRecord) {
     if (
-      ('path' in node) ||
-      ('stem' in node) ||
-      (('baseDir' in node) &&
+      ('baseDir' in node) &&
       ('extensions' in node) &&
-      ('suffix'))
+      (('suffix' in node) || ('stem' in node))
     ) {
       rulesRecord[path] = false
       return
     }
+    //recognize that some objects required or recommended by the spec are directories
+    if (
+      'path' in node &&
+      'directory' in node
+    ){
+      rulesRecord[path] = false
+      return
+    }
     else {
       Object.keys(node).map((key) => {
         if(
@@ -88,8 +94,11 @@ function findRuleMatches(schema, context) {
     context.filenameRules.length === 0 &&
     context.file.path !== '/.bidsignore'
   ) {
+    //if no rules are found to match given file/directory, add NotIncluded warning to indicate 
+    //that the file/directory is not part of the PsychDS specification
     context.issues.addSchemaIssue('NotIncluded', [context.file])
     if(context.file.name === "dataset_description.json"){
+      //if global metadata file is located outside of root directory, issue specific warning
       context.issues.addSchemaIssue(
         "WrongMetadataLocation",
         [context.file],
@@ -101,28 +110,55 @@ function findRuleMatches(schema, context) {
   return Promise.resolve()
 }
 
+function checkFileRules(arbitraryNesting: boolean, hasSuffix: boolean, node, context){
+  let baseDirCond: boolean = null
+  let suffixStemCond: boolean = null
+
+  //if arbitraryNesting applies, then it is only required that the file is located in the correct base directory,
+  //with any number of subdirectories intervening
+  if (arbitraryNesting)
+    baseDirCond = context.baseDir === node.baseDir
+  //otherwise, the file must be located directly under the baseDir
+  else{
+    //if the baseDir is root, arbitraryNesting does not apply
+    if(context.baseDir === "/")
+      baseDirCond = context.path === `/${context.file.name}`
+    else
+      baseDirCond = context.path === `/${node.baseDir}/${context.file.name}`
+  }
+
+  //if the suffix property is present on a rule, then the file should be identified by its suffix
+  if (hasSuffix)
+    suffixStemCond = context.suffix === node.suffix
+  //otherwise, a file should be identified with its stem
+  else
+    suffixStemCond = context.file.name.startsWith(node.stem)
+
+  //files are identified by a combination of their baseDir, their extensions, and either their stem or their suffix
+  if (
+    baseDirCond &&
+    node.extensions.includes(context.extension) &&
+    suffixStemCond
+  )
+    return true
+  else
+    return false
+}
+
 /* Schema rules specifying valid filenames follow a variety of patterns.
- * 'path', 'stem' or 'suffixies' contain the most unique identifying
+ * 'baseDir', 'extensions', 'stem' or 'suffixies' contain the most unique identifying
  * information for a rule. We don't know what kind of filename the context is,
- * so if one of these three match the respective value in the context lets
+ * so if one of these  match the respective value in the context lets
  * assume that this schema rule is applicable to this file.
  */
 export function _findRuleMatches(node, path, context) {
-  //console.log(node)
-  if (
-    ('path' in node && context.file.path === node.path) ||
-    ('stem' in node && context.file.name.startsWith(node.stem)) ||
-    ((('baseDir' in node && 'arbitraryNesting' in node && node.arbitraryNesting && context.baseDir === node.baseDir) ||
-    ('baseDir' in node && 'arbitraryNesting' in node && !node.arbitraryNesting && context.path === `/${node.baseDir}/${context.file.name}`)) &&
-    ('extensions' in node && node.extensions.includes(context.extension)) &&
-    ('suffix' in node && context.suffix === node.suffix))
-  ) {
-    context.filenameRules.push(path)
-    return
+  if ('arbitraryNesting' in node){
+    if (checkFileRules(node.arbitraryNesting,'suffix' in node, node, context)){
+      context.filenameRules.push(path)
+      return
+    }
   }
-  if (
-    !('path' in node || 'stem' in node || 'baseDir' in node || 'extensions' in node || 'suffix' in node)
-  ) {
+  else {
     Object.keys(node).map((key) => {
       if(
         typeof node[key] === 'object'