Merge pull request diffblue#154 from diffblue/update-merge

Update for latest merge of master

Author: NathanJPhillips

cbmc

@@ -0,0 +1,34 @@
+#! /usr/bin/env python
+
+from reformat_docs import convert_file
+from os import walk
+from os.path import join
+from re import match
+import sys
+
+
+"""
+Run this from test-gen's top-level directory.
+"""
+
+def main():
+    if len(sys.argv) < 2:
+        print "Pass the folder to convert on the command line"
+        sys.exit(1)
+
+    IGNORE_LIST = []
+
+    MATCH_EXPR = r'.*\.(h|cpp)'
+
+    for root, dirs, files in walk(sys.argv[1]):
+        for file in files:
+            path = join(root, file)
+            if any(map(lambda x: match(x, path), IGNORE_LIST)):
+                print 'ignoring', path
+                continue
+            if not match(MATCH_EXPR, path):
+                continue
+            convert_file(path, True)
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/do_doc_convert.py

@@ -0,0 +1,8 @@
+#!/usr/bin/env sh
+
+script_dir="$(dirname "$0")"
+
+python "${script_dir}/reformat_docs.py" "$1" -i
+python "${script_dir}/reformat_docs.py" "$2" -i
+python "${script_dir}/reformat_docs.py" "$3" -i
+git merge-file "$1" "$2" "$3"

scripts/doxy_merge_driver.sh

@@ -0,0 +1,315 @@
+#! /usr/bin/env python
+
+import re, collections, textwrap, sys, argparse, platform, os
+
+Field = collections.namedtuple('Field', ['name', 'contents'])
+
+Header = collections.namedtuple('Header', ['module'])
+
+Function = collections.namedtuple('Function',
+        ['name', 'purpose', 'remarks', 'template_params', 'inputs', 'returns'])
+
+Class = collections.namedtuple('Class', ['name', 'purpose', 'remarks', 'template_params'])
+
+
+def warn(message):
+    """ Print a labelled message to stderr.  """
+    sys.stderr.write('Warning: %s\n' % message)
+
+
+def header_from_block(file, block):
+    """ Create a Header structure from a parsed Block.  """
+    paragraphs = []
+    module = block.fields['Module'].strip()
+    if not os.path.basename(file).startswith(module):
+        paragraphs.append(module)
+    purpose = block.fields.get('Purpose', None)
+    if purpose:
+        end_of_purpose = purpose.find('@ Copyright')
+        if end_of_purpose == -1:
+            end_of_purpose = len(purpose)
+        paragraphs.append(purpose[ : end_of_purpose ].strip())
+    date = block.fields.get('Date', '')
+    end_of_first_date_line = date.find('\n')
+    if end_of_first_date_line != -1:
+        paragraphs.append(date[ end_of_first_date_line+1 : ].strip())
+    return Header('\n\n'.join(paragraphs))
+
+def function_from_block(block):
+    """ Create a Function structure from a parsed Block.  """
+    return Function(
+        block.fields.get('Function', None),
+        block.fields.get('Purpose', None),
+        block.fields.get('Remarks', None),
+        block.fields.get('Template parameters', None),
+        block.fields.get('Inputs', None),
+        block.fields.get('Outputs', block.fields.get('Output', None)))
+
+def class_from_block(block):
+    """ Create a Class structure from a parsed Block.  """
+    return Class(
+        block.fields.get('Class', None),
+        block.fields.get('Purpose', None),
+        block.fields.get('Remarks', None),
+        block.fields.get('Template parameters', None))
+
+FIELD_PARSE_EXPR = r'\n *Purpose:(.*?)(?=\Z|\n\s*\n\s*(?!NOTE:)(?:[a-zA-Z0-9]+|Template parameters)\s*:)|\n *Date:(.*?)(?=@ Copyright)|\n *([a-zA-Z0-9]+|Template parameters)\s*:\n?(.*?)^$'
+
+def remove_extra_space(text):
+    return '\n'.join([line.strip() for line in re.sub(r'  +', ' ', text).splitlines()])
+
+def fill(wrapper, text):
+    return '\n'.join([
+            wrapper.fill(match.group(1))
+            for match
+            in re.finditer(r'(.*?)(?:^\s*$|\Z)', text, re.MULTILINE | re.DOTALL)
+        ]).strip()
+
+def parse_fields(block_contents):
+    """ Extract the named fields of an old-style comment block.  """
+
+    field_re = re.compile(FIELD_PARSE_EXPR, re.MULTILINE | re.DOTALL)
+    for match in field_re.finditer(block_contents):
+        if match.lastindex == 1:
+            # The field is a Purpose field
+            yield Field('Purpose', remove_extra_space(match.group(1)))
+        elif match.lastindex == 2:
+            # The field is a Date field
+            yield Field('Date', remove_extra_space(match.group(2)))
+        elif match.group(3):
+            # The field is any other field
+            yield Field(match.group(3), remove_extra_space(match.group(4)))
+
+
+Block = collections.namedtuple('Block', ['fields'])
+
+
+def has_field(block, field_name):
+    """ Return whether the block has a field with the given name.  """
+    return field_name in block.fields
+
+
+def make_doxy_comment(text):
+    text = re.sub(r'^(?!$)', r'/// ', text, flags=re.MULTILINE)
+    text = re.sub(r'^(?=$)', r'///', text, flags=re.MULTILINE)
+    return text
+
+
+class GenericFormatter(object):
+    def __init__(self, doc_width):
+        self.text_wrapper = textwrap.TextWrapper(width=doc_width)
+        self.indented_wrapper = textwrap.TextWrapper(width=doc_width,
+                subsequent_indent=r'  ')
+
+    def convert(self, block):
+        sections = filter(None, self.convert_sections(block))
+        if sections:
+            return make_doxy_comment('\n'.join(sections)) + '\n'
+        return ''
+
+HEADER_TEMPLATE = r'// Copyright 2016-2017 DiffBlue Limited. All Rights Reserved.'
+
+class HeaderFormatter(GenericFormatter):
+    def format_module(self, header):
+        if not header.module:
+            return None
+
+        # The file directive must be followed by a newline in order to refer to
+        # the current file
+        return '\\file\n' + fill(self.indented_wrapper, header.module)
+
+    def is_block_valid(self, block):
+        return has_field(block, 'Module')
+
+    def convert_sections(self, block):
+        return [self.format_module(block)]
+
+
+class FunctionFormatter(GenericFormatter):
+    def __init__(self, doc_width):
+        super(FunctionFormatter, self).__init__(doc_width)
+
+    def format_purpose(self, block):
+        return fill(self.text_wrapper, block.purpose) if block.purpose else None
+
+    def format_remarks(self, block):
+        return fill(
+            self.indented_wrapper,
+            r'\remark %s' % block.remarks) if block.remarks else None
+
+    def format_template_params(self, function):
+        if not function.template_params:
+            return None
+
+        def param_replacement(match):
+            return r'\tparam %s: ' % match.group(1)
+
+        text = re.sub(r'\n(?![a-zA-Z0-9_`]+\s*[:-])', ' ', function.template_params, flags=re.MULTILINE)
+        text, num_replacements = re.subn(
+            r'^([a-zA-Z0-9_`]+)\s*[:-]\s*',
+            param_replacement,
+            text,
+            flags=re.MULTILINE)
+
+        if num_replacements == 0:
+            return None
+
+        return '\n'.join(fill(self.indented_wrapper, param) for param in text.split('\n'))
+
+    def format_inputs(self, function):
+        if not function.inputs:
+            return None
+
+        if re.match(r'^\s*\S+\s*$', function.inputs) or function.inputs == 'See purpose':
+            return None
+
+        def param_replacement(match):
+            return r'\param %s: ' % match.group(1)
+
+        text = re.sub(r'\n(?![a-zA-Z0-9_`]+\s*[:-])', ' ', function.inputs, flags=re.MULTILINE)
+        text, num_replacements = re.subn(
+            r'^([a-zA-Z0-9_`]+)\s*[:-]\s*',
+            param_replacement,
+            text,
+            flags=re.MULTILINE)
+
+        if num_replacements == 0:
+            text = r'\par parameters: %s' % text
+
+        return '\n'.join(fill(self.indented_wrapper, param) for param in text.split('\n'))
+
+    def format_returns(self, function):
+        if not function.returns:
+            return None
+        if function.returns == 'See purpose':
+            return None
+        return fill(self.indented_wrapper, r'\return %s' % function.returns)
+
+    def is_block_valid(self, block):
+        return has_field(block, 'Function')
+
+    def convert_sections(self, block):
+        return [
+            self.format_purpose(block),
+            self.format_remarks(block),
+            self.format_template_params(block),
+            self.format_inputs(block),
+            self.format_returns(block)
+        ]
+
+
+class ClassFormatter(GenericFormatter):
+    def __init__(self, doc_width):
+        super(ClassFormatter, self).__init__(doc_width)
+
+    def format_purpose(self, block):
+        return fill(self.text_wrapper, block.purpose) if block.purpose else None
+
+    def format_remarks(self, block):
+        return fill(
+            self.indented_wrapper,
+            r'\remark %s' % block.remarks) if block.remarks else None
+
+    def format_template_params(self, function):
+        if not function.template_params:
+            return None
+
+        def param_replacement(match):
+            return r'\tparam %s: ' % match.group(1)
+
+        text = re.sub(r'\n(?![a-zA-Z0-9_`]+\s*[:-])', ' ', function.template_params, flags=re.MULTILINE)
+        text, num_replacements = re.subn(
+            r'^([a-zA-Z0-9_`]+)\s*[:-]\s*',
+            param_replacement,
+            text,
+            flags=re.MULTILINE)
+
+        if num_replacements == 0:
+            return None
+
+        return '\n'.join(fill(self.indented_wrapper, param) for param in text.split('\n'))
+
+    def is_block_valid(self, block):
+        return has_field(block, 'Class')
+
+    def convert_sections(self, block):
+        return [
+            self.format_purpose(block),
+            self.format_remarks(block),
+            self.format_template_params(block)
+        ]
+
+
+def replace_block(
+        block_contents,
+        file_contents,
+        file,
+        header_formatter,
+        class_formatter,
+        function_formatter):
+    """
+    Replace an old-style documentation block with the doxygen equivalent
+    """
+    block = Block(
+            {f.name: f.contents for f in parse_fields(block_contents.group(1))})
+
+    if header_formatter.is_block_valid(block):
+        converted = header_formatter.convert(header_from_block(file, block))
+        return HEADER_TEMPLATE + '\n\n' + converted + '\n'
+
+    if class_formatter.is_block_valid(block):
+        return class_formatter.convert(class_from_block(block))
+
+    if function_formatter.is_block_valid(block):
+        return function_formatter.convert(function_from_block(block))
+
+    warn('block in "%s" has unrecognised format:\n%s' %
+            (file, block_contents.group(1)))
+
+    return ''
+
+
+def convert_file(file, inplace):
+    """ Replace documentation in file with doxygen-styled comments.  """
+    with open(file) as f:
+        contents = f.read()
+
+    doc_width = 76
+
+    header_formatter = HeaderFormatter(doc_width)
+    class_formatter = ClassFormatter(doc_width)
+    function_formatter = FunctionFormatter(doc_width)
+
+    block_re = re.compile(
+            r'^/\*+\\$(.*?)^\\\*+/$\s*', re.MULTILINE | re.DOTALL)
+    new_contents = block_re.sub(
+            lambda match: replace_block(
+                match,
+                contents,
+                file,
+                header_formatter,
+                class_formatter,
+                function_formatter), contents)
+
+    if inplace:
+        with open(file, 'w') as f:
+            f.write(new_contents)
+    else:
+        sys.stdout.write(new_contents)
+
+
+def main():
+    """ Run convert_file from the command-line.  """
+    parser = argparse.ArgumentParser()
+    parser.add_argument('file', type=str, help='The file to process')
+    parser.add_argument('-i', '--inplace', action='store_true',
+            help='Process in place')
+    args = parser.parse_args()
+
+    convert_file(args.file, args.inplace)
+
+    return 0
+
+
+if __name__ == '__main__':
+    sys.exit(main())

scripts/reformat_docs.py

@@ -1,27 +1,12 @@
-/*******************************************************************\
+// Copyright 2016-2017 DiffBlue Limited. All Rights Reserved.
 
-Module: Security Driver Main Module
-
-Author: Chris Smowton, [email protected]
-
-\*******************************************************************/
+/// \file
+/// Security Driver Main Module
 
 #include <util/unicode.h>
 
 #include "sec_driver_parse_options.h"
 
-/*******************************************************************\
-
-Function: main / wmain
-
-  Inputs:
-
- Outputs:
-
- Purpose:
-
-\*******************************************************************/
-
 #ifdef _MSC_VER
 int wmain(int argc, const wchar_t **argv_wide)
 {