ENH: added new parameter --process-batch that allows to validate all …

…process files in certain sub-trees of the file system. Failing to parse one of the process files no longer results in an aborted script, just triggers return code of 1 and then continues with the next one.

validation/mdvalid.py

@@ -16,6 +16,7 @@
 import platform as platf
 import argparse as argparse
 import logging as logging
+import fnmatch as fnm
 import traceback as trb
 import difflib as diffl
 import collections as col
@@ -33,7 +34,7 @@
 
 __author__ = 'Peter Ebert'
 __contact__ = 'pebert@mpi-inf.mpg.de'
-__version__ = '0.2'
+__version__ = '0.3'
 
 # ==================================================================
 # Configuration
@@ -173,6 +174,11 @@ def get_cmdline_args():
     parser.add_argument('--annoying', dest='annoying', action='store_true', default=False,
                         help='Complain about empty comment fields in the process XML. Note that empty comments'
                              ' will only result in errors and a non-zero exit status if --strict is set.')
+    parser.add_argument('--process-batch', dest='processbatch', action='store_true', default=False,
+                        help='If set, the parameter(s) specified via --process/-p are assumed to be'
+                             ' top-level folders. The script will walk down these folders and collect'
+                             ' all .xml files and validate these. Note that validating analysis metadata'
+                             ' files against the collected process file(s) is not supported.')
     parser.add_argument('--no-wildcards', '-n', dest='nowildcards', action='store_true', default=False,
                         help='If set, it is assumed that each placeholder in the command lines of the process'
                              ' XML is matched exactly to a filetype identifier or parameter identifier in'
@@ -184,6 +190,26 @@ def get_cmdline_args():
     return args
 
 
+def collect_process_files(topdir):
+    """
+    :param topdir:
+    :return:
+    """
+
+    def err(x):
+        raise OSError('Error encountered at {} - original error: {}'.format(x.filename, str(x)))
+
+    processes = []
+    for root, dirs, files in os.walk(topdir,
+                                     followlinks=False,
+                                     onerror=err):
+        if files:
+            xml_files = fnm.filter(files, '*.xml')
+            for fn in xml_files:
+                processes.append(os.path.join(root, fn))
+    return processes
+
+
 def check_file_identifiers(process, amd_section, xmlpath, checked):
     """
     Check for (case-insensitive) exact matches between the filetype
@@ -476,8 +502,12 @@ def parse_xml(filepath, xmlparser):
         return process
     except (XMLSyntaxError, OSError) as err:
         errlog().error('Cannot parse XML process file: {}'.format(filepath))
+        errlog().error('Error: {} - code {}'.format(err.msg, err.code))
         errlog().error('Is the file readable with your user account?')
         errlog().error('Is the file an XML version 1 file?')
+        errlog().error('Does the file contain invalid characters?'
+                       ' Please run "python mdvalid.py --help"'
+                       ' to see how to escape special characters in XML.')
         raise err
 
 
@@ -520,30 +550,37 @@ def validate_files(files, schema, encoding, args):
     :return: zero if all files are valid
      :rtype: int
     """
+    retcode = 0
     xmlfile, amdfile = files
     xmlparser = etree.XMLParser(encoding=encoding, strip_cdata=False)
     outlog().info('Checking: [XML] {} | [AMD] {}'.format(os.path.basename(xmlfile), os.path.basename(amdfile)))
-    process = parse_xml(xmlfile, xmlparser)
-    processname, processver = get_process_info(process)
-    retcode = 0
-    if schema.validate(process):
-        errlog().debug('XML file {} validates against schema'.format(os.path.basename(xmlfile)))
-        if args.annoying:
-            for c in COMMENTS_IN:
-                errlog().debug('Checking for comments')
-                retcode |= check_comments(process, AMD_TO_XML_COM[c], args.strict)
-        if amdfile:
-            analysis = parse_amd(amdfile, args.strict)
-            rc = check_analysis_metadata(process, processname, processver, analysis, args)
-            if rc == 1:
-                outlog().error('AMD file {} does not validate against process XML'.format(os.path.basename(amdfile)))
-            retcode |= rc
-    else:
-        errorlog = schema.error_log
-        errlog().error('XML file {} not valid: {}'.format(os.path.basename(xmlfile), errorlog.last_error))
+    try:
+        process = parse_xml(xmlfile, xmlparser)
+    except Exception:
         retcode = 1
-    outlog().info('Complete: [XML] {} | [AMD] {}'.format(os.path.basename(xmlfile), os.path.basename(amdfile)))
-    return retcode
+    else:
+        processname, processver = get_process_info(process)
+        retcode = 0
+        if schema.validate(process):
+            errlog().debug('XML file {} validates against schema'.format(os.path.basename(xmlfile)))
+            if args.annoying:
+                for c in COMMENTS_IN:
+                    errlog().debug('Checking for comments')
+                    retcode |= check_comments(process, AMD_TO_XML_COM[c], args.strict)
+            if amdfile:
+                analysis = parse_amd(amdfile, args.strict)
+                rc = check_analysis_metadata(process, processname, processver, analysis, args)
+                if rc == 1:
+                    outlog().error('AMD file {} does not validate against process XML'.format(os.path.basename(amdfile)))
+                retcode |= rc
+        else:
+            errorlog = schema.error_log
+            errlog().error('XML file {} not valid: {}'.format(os.path.basename(xmlfile), errorlog.last_error))
+            retcode = 1
+    finally:
+        status = 'FAIL' if retcode > 0 else 'OK'
+        outlog().info('{}: [XML] {} | [AMD] {}'.format(status, os.path.basename(xmlfile), os.path.basename(amdfile)))
+        return retcode
 
 
 def create_schema(schema_path):
@@ -584,6 +621,16 @@ def run():
         # works only in Python 3.x
         ### for t in itertools.zip_longest(args.process, args.analysis, fillvalue='')
         # so we make the cases explicit
+        if args.processbatch:
+            processes = []
+            for path in args.process:
+                assert os.path.isdir(path), 'This path is not a directory: {} -' \
+                                            ' please specify only directories in --process-batch' \
+                                            ' mode'.format(path)
+                subtree_proc = collect_process_files(path)
+                processes.extend(subtree_proc)
+            setattr(args, 'process', sorted(processes))
+            setattr(args, 'analysis', [])
         if len(args.process) == len(args.analysis):
             checklist = zip(args.process, args.analysis)
         elif len(args.process) == 1 and len(args.analysis) > 1: