From 21aab58a4fee11fcfaa03623710bcf5cba37bd2e Mon Sep 17 00:00:00 2001 From: Frederik Deweerdt Date: Tue, 14 Mar 2017 08:55:11 -0700 Subject: [PATCH 1/5] extract https://github.com/deweerdt/docoment @ 4fcaa60 () at deps/docoment --- deps/docoment/README.md | 39 ++++++ deps/docoment/docofile.example | 13 ++ deps/docoment/docoment.py | 227 +++++++++++++++++++++++++++++++ deps/docoment/logo.svg | 81 +++++++++++ deps/docoment/templates/func.tpl | 14 ++ 5 files changed, 374 insertions(+) create mode 100644 deps/docoment/README.md create mode 100644 deps/docoment/docofile.example create mode 100755 deps/docoment/docoment.py create mode 100644 deps/docoment/logo.svg create mode 100644 deps/docoment/templates/func.tpl diff --git a/deps/docoment/README.md b/deps/docoment/README.md new file mode 100644 index 0000000..1885957 --- /dev/null +++ b/deps/docoment/README.md @@ -0,0 +1,39 @@ +

+ Docoment Logo +

+ +# Overview +Docoment extracts comments from C code and generates documentation. + +# Requirements + +Docoment requires python 2, and the following packages: + +``` +$ sudo pip install jinja2 clang +``` + +# How to use + +- Create a docofile +``` +[project] +name = project-name +path = /src + /include +extra_args = -I /include +files = *.c + +[output] +json = true +html = true + +[templates] +path = ./templates + +``` + +- Run docoment +``` +python docoment.py +``` diff --git a/deps/docoment/docofile.example b/deps/docoment/docofile.example new file mode 100644 index 0000000..9a656c6 --- /dev/null +++ b/deps/docoment/docofile.example @@ -0,0 +1,13 @@ +[project] +name = project-name +path = /src + /include +extra_args = -I /include +files = *.c + +[output] +json = true +html = true + +[templates] +path = ./templates diff --git a/deps/docoment/docoment.py b/deps/docoment/docoment.py new file mode 100755 index 0000000..afb5c36 --- /dev/null +++ b/deps/docoment/docoment.py @@ -0,0 +1,227 @@ +#!/usr/bin/env python2 +""" Usage: call with +""" + +import os +import re +import sys +import json +import shlex +import jinja2 +import fnmatch +import subprocess +import clang.cindex +import ConfigParser + + +def comment_to_dict(location, comment): + brief = [] + result = {} + + def _add_param(name, value): + if 'params' not in result: + result['params'] = {} + elif name in result['params']: + print("Warning: In %s, param %s already documented." % (location, name)) + result['params'][name] = value + + for line in comment.split('\n'): + line = line.lstrip('/*< ').rstrip('*/ ') + if line: + if line.startswith('@param'): + line = line[6:].lstrip() + try: + name, desc = line.split(None, 1) + _add_param(name, desc.strip()) + except ValueError: + print("Warning: Could not extract param from: %s" % line) + elif line.startswith('@'): + key, value = line[1:].split(None, 1) + if key in result: + print("Warning: In %s, %s already documented." % (location, key)) + result[key] = value.lstrip() + else: + brief.append(line) + if brief: + result['brief'] = '\n'.join(brief) + return result + + +class Docoment(object): + + def __init__(self, config_file="docofile"): + self.definitions = {} + self.decl_types = { + clang.cindex.CursorKind.TYPE_ALIAS_DECL: None, + clang.cindex.CursorKind.MACRO_DEFINITION: None, + clang.cindex.CursorKind.TYPEDEF_DECL: None, + clang.cindex.CursorKind.ENUM_DECL: None, + clang.cindex.CursorKind.UNION_DECL: None, + clang.cindex.CursorKind.FUNCTION_DECL: None, + clang.cindex.CursorKind.STRUCT_DECL: None, + } + self.index = clang.cindex.Index.create() + + config = ConfigParser.ConfigParser() + config.read(config_file) + self.project = config.get('project', 'name') + self.paths = shlex.split(config.get('project', 'path')) + self.patterns = shlex.split(config.get('project', 'files')) + self.output_json = config.getboolean('output', 'json') if config.has_option('output', 'json') else True + self.output_html = config.getboolean('output', 'html') if config.has_option('output', 'html') else True + self.templates = config.get('html', 'templates') if config.has_option('html', 'templates') else './templates' + self.extra_args = self._get_default_includes() + if config.has_option('project', 'extra_args'): + self.extra_args.extend(shlex.split(config.get('project', 'extra_args'))) + if os.environ.get('CFLAGS') is not None: + self.extra_args.extend(shlex.split(os.environ.get('CFLAGS'))) + self._register_hooks() + + def _get_default_includes(self): + regex = re.compile(ur'(?:\#include \<...\> search starts here\:)(?P.*?)(?:End of search list)', re.DOTALL) + process = subprocess.Popen(['clang', '-v', '-E', '-x', 'c', '-'], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + process_out, process_err = process.communicate('') + output = process_out + process_err + includes = [] + for p in re.search(regex, output).group('list').split('\n'): + p = p.strip() + if len(p) > 0 and p.find('(framework directory)') < 0: + includes.append('-isystem') + includes.append(p) + return includes + + def is_included(self, location): + if not location.file: + return False + for path in self.paths: + if path in location.file.name: + return True + return False + + def record_definition(self, node): + usr = node.get_usr() + if usr and node.kind in self.decl_types: + if usr not in self.definitions: + self.definitions[usr] = { + 'kind': node.kind.name, + 'spelling': node.spelling, + 'location': { + 'line': node.location.line, + 'column': node.location.column + } + } + if node.location.file: + self.definitions[usr]['location']['file'] = node.location.file.name + if node.raw_comment: + self.definitions[usr]['comment'] = comment_to_dict(node.location, node.raw_comment) + func = self.decl_types[node.kind] + if func: + info = func(node) + if info: + self.definitions[usr].update(info) + + def _register_hooks(self): + def _type_id(ctype): + decl = ctype.get_declaration() + if decl.kind == clang.cindex.CursorKind.NO_DECL_FOUND: + return None + if not self.is_included(decl.location): + return None + return decl.get_usr() + + def _type_to_dict(ctype): + spelling = ctype.spelling + suffix = '' + while ctype.kind == clang.cindex.TypeKind.POINTER: + suffix += '*' + ctype = ctype.get_pointee() + return {'type': _type_id(ctype), 'type_spelling': spelling} + + def _func_to_dict(node): + params = [] + for param in node.get_arguments(): + p = _type_to_dict(param.type) + p['spelling'] = param.spelling + params.append(p) + return {'params': params, 'result': _type_to_dict(node.result_type)} + self.decl_types[clang.cindex.CursorKind.FUNCTION_DECL] = _func_to_dict + + def _struct_to_dict(node): + fields = [] + for field in node.type.get_fields(): + p = _type_to_dict(field.type) + p['spelling'] = field.spelling + if field.raw_comment: + p['comment'] = comment_to_dict(field.location, field.raw_comment) + fields.append(p) + return {'fields': fields} + self.decl_types[clang.cindex.CursorKind.STRUCT_DECL] = _struct_to_dict + + def _enum_to_dict(node): + fields = [] + for field in node.get_children(): + if field.kind == clang.cindex.CursorKind.ENUM_CONSTANT_DECL: + p = {'spelling': field.spelling, 'value': field.enum_value} + if field.raw_comment: + p['comment'] = comment_to_dict(field.location, field.raw_comment) + fields.append(p) + return {'fields': fields} + self.decl_types[clang.cindex.CursorKind.ENUM_DECL] = _enum_to_dict + + def _typedef_to_dict(node): + return {'canonical': _type_to_dict(node.type.get_canonical())} + self.decl_types[clang.cindex.CursorKind.TYPEDEF_DECL] = _typedef_to_dict + + def _parse_file(self, path): + print("Parsing %s" % path) + tu = self.index.parse(path, args=self.extra_args, + options=clang.cindex.TranslationUnit.PARSE_DETAILED_PROCESSING_RECORD) + for diagnostic in tu.diagnostics: + if diagnostic.severity > clang.cindex.Diagnostic.Warning: + print(diagnostic) + sys.exit(-1) + nodes = [tu.cursor] + while len(nodes): + node = nodes.pop() + if self.is_included(node.location): + if node.is_definition() or node.kind == clang.cindex.CursorKind.MACRO_DEFINITION: + self.record_definition(node) + else: + nodes.extend([c for c in node.get_children()]) + elif node.kind == clang.cindex.CursorKind.TRANSLATION_UNIT: + nodes.extend([c for c in node.get_children()]) + + def generate_json(self, path): + with open(path, "w") as jf: + json.dump(self.definitions, jf, indent=True, sort_keys=True) + + def generate_html(self, path): + env = jinja2.Environment( + loader=jinja2.FileSystemLoader(self.templates) + ) + template = env.get_template('func.tpl') + with open(path, 'w') as html: + for usr in self.definitions.keys(): + e = self.definitions[usr] + if e['kind'] == clang.cindex.CursorKind.FUNCTION_DECL.name: + html.write(template.render(func=e) + '\n') + + def run(self): + for path in self.paths: + if os.path.isdir(path): + for root, dirs, files in os.walk(path): + for pattern in self.patterns: + for curfile in fnmatch.filter(files, pattern): + self._parse_file(os.path.join(root, curfile)) + else: + for pattern in self.patterns: + if fnmatch.fnmatch(path, pattern): + self._parse_file(path) + if self.output_json: + self.generate_json('result.json') + if self.output_html: + self.generate_html('index.html') + +doc = Docoment() +doc.run() diff --git a/deps/docoment/logo.svg b/deps/docoment/logo.svg new file mode 100644 index 0000000..ac21d2c --- /dev/null +++ b/deps/docoment/logo.svg @@ -0,0 +1,81 @@ + + + + + + + + + + image/svg+xml + + + + + + + /* docoment */ + + diff --git a/deps/docoment/templates/func.tpl b/deps/docoment/templates/func.tpl new file mode 100644 index 0000000..7391d83 --- /dev/null +++ b/deps/docoment/templates/func.tpl @@ -0,0 +1,14 @@ +
+{% if func['comment'] %} +
+ {{ func['comment']['brief'] }} +
+{% endif %} +{{ func['result']['type_spelling'] }} {{ func['spelling'] }}({% if func['params'] %} +
    +{% for param in func['params'] %} +
  • {{ param['type_spelling'] }} {{ param['spelling'] }}{% if func['comment'] %}: {{ func['comment']['params'][param['spelling']] }}{% endif %}
    • +{% endfor %} +
+{% endif %}) +
From b18b07ac5014f5f3345d2a5f39d096867b12eb1b Mon Sep 17 00:00:00 2001 From: Frederik Deweerdt Date: Tue, 14 Mar 2017 10:15:42 -0700 Subject: [PATCH 2/5] extract https://github.com/deweerdt/docoment @ c796ec486f () at deps/docoment --- deps/docoment/docofile.example | 4 ++-- deps/docoment/docoment.py | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/deps/docoment/docofile.example b/deps/docoment/docofile.example index 9a656c6..48eda9c 100644 --- a/deps/docoment/docofile.example +++ b/deps/docoment/docofile.example @@ -9,5 +9,5 @@ files = *.c json = true html = true -[templates] -path = ./templates +[html] +templates = ./templates diff --git a/deps/docoment/docoment.py b/deps/docoment/docoment.py index afb5c36..1fffe19 100755 --- a/deps/docoment/docoment.py +++ b/deps/docoment/docoment.py @@ -9,6 +9,7 @@ import shlex import jinja2 import fnmatch +import argparse import subprocess import clang.cindex import ConfigParser @@ -223,5 +224,10 @@ def run(self): if self.output_html: self.generate_html('index.html') -doc = Docoment() -doc.run() +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("-c", "--config", help="Path to the docofile", default=None) + args = parser.parse_args() + + doc = Docoment(args.config) + doc.run() From 6560ce40e09b407e52095aaec441f5addf4463d1 Mon Sep 17 00:00:00 2001 From: Frederik Deweerdt Date: Tue, 14 Mar 2017 10:15:59 -0700 Subject: [PATCH 3/5] Doc generation, wip --- CMakeLists.txt | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e0328a..d523ef2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,7 +78,7 @@ IF (NOT WITH_MRUBY_INCLUDE OR NOT WITH_MRUBY_LIB) MRUBY_BUILD_DIR=${CMAKE_CURRENT_BINARY_DIR}/mruby ruby minirake WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/deps/mruby-1.2.0) - TARGET_INCLUDE_DIRECTORIES(h2get BEFORE PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/deps/mruby-1.2.0/include) + INCLUDE_DIRECTORIES(${CMAKE_CURRENT_SOURCE_DIR}/deps/mruby-1.2.0/include) # note: the paths need to be determined before libmruby.flags.mak is generated TARGET_LINK_LIBRARIES(h2get "${CMAKE_CURRENT_BINARY_DIR}/mruby/host/lib/libmruby.a" m) ADD_DEPENDENCIES(h2get mruby) @@ -89,3 +89,17 @@ ENDIF () INSTALL(TARGETS h2get RUNTIME DESTINATION bin) + +# +# Docs +# +GET_PROPERTY(dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES) +FOREACH(dir ${dirs}) + SET(INCL_DIRS "${INCL_DIRS} -I${dir}") +ENDFOREACH() + + +ADD_CUSTOM_TARGET(docs env CFLAGS=${INCL_DIRS} ${CMAKE_CURRENT_SOURCE_DIR}/deps/docoment/docoment.py -c ${CMAKE_CURRENT_SOURCE_DIR}/srcdoc/docofile + WORKING_DIRECTORY ${CMAKE_BINARY_DIR}) + + From 94ac9035af4fba6bce2269f3a12282d79270444f Mon Sep 17 00:00:00 2001 From: Frederik Deweerdt Date: Tue, 14 Mar 2017 10:17:57 -0700 Subject: [PATCH 4/5] Add srcdoc --- srcdoc/docofile | 12 ++++++++++++ srcdoc/func.tpl | 8 ++++++++ 2 files changed, 20 insertions(+) create mode 100644 srcdoc/docofile create mode 100644 srcdoc/func.tpl diff --git a/srcdoc/docofile b/srcdoc/docofile new file mode 100644 index 0000000..4fe430a --- /dev/null +++ b/srcdoc/docofile @@ -0,0 +1,12 @@ +[project] +name = h2get +path = src + include +files = *.c + +[output] +json = false +html = true + +[html] +templates = ./srcdoc/ diff --git a/srcdoc/func.tpl b/srcdoc/func.tpl new file mode 100644 index 0000000..80887c9 --- /dev/null +++ b/srcdoc/func.tpl @@ -0,0 +1,8 @@ +
+{% if func['comment'] %} +
+ {{ func['comment']['brief'] }} +
+{% endif %} +{{ func['result']['type_spelling'] }} {{ func['spelling'] }} +
From e29176e3cbb8bf619e26a08f56bbc81098c7551f Mon Sep 17 00:00:00 2001 From: Frederik Deweerdt Date: Tue, 14 Mar 2017 10:38:48 -0700 Subject: [PATCH 5/5] extract https://github.com/deweerdt/docoment @ ebc5c68ff3 () at deps/docoment --- deps/docoment/docoment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/docoment/docoment.py b/deps/docoment/docoment.py index 1fffe19..f3a8bad 100755 --- a/deps/docoment/docoment.py +++ b/deps/docoment/docoment.py @@ -74,7 +74,7 @@ def __init__(self, config_file="docofile"): self.extra_args = self._get_default_includes() if config.has_option('project', 'extra_args'): self.extra_args.extend(shlex.split(config.get('project', 'extra_args'))) - if os.environ.get('CFLAGS') is not None: + if os.environ.get('CFLAGS'): self.extra_args.extend(shlex.split(os.environ.get('CFLAGS'))) self._register_hooks()