From 5e64fa9c441ff273f39527f4014d3c3e724d321a Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 19 May 2016 20:32:48 +0300 Subject: [PATCH 01/38] kernel-doc/rst: fix use of uninitialized value I'm not quite sure why the errors below are happening, but this fixes them. Use of uninitialized value in string ne at ./scripts/kernel-doc line 1819, line 6494. Use of uninitialized value $_[0] in join or string at ./scripts/kernel-doc line 1759, line 6494. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 2fc8fad5195e8..babb374c043dd 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1803,7 +1803,8 @@ sub output_function_rst(%) { } else { print " ``$parameter``\n"; } - if ($args{'parameterdescs'}{$parameter_name} ne $undescribed) { + if (defined($args{'parameterdescs'}{$parameter_name}) && + $args{'parameterdescs'}{$parameter_name} ne $undescribed) { my $oldprefix = $lineprefix; $lineprefix = " "; output_highlight_rst($args{'parameterdescs'}{$parameter_name}); From 86ae2e38d40ed6bf7c907d126053e6f4b1dc5b6e Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 21 Jan 2016 13:05:22 +0200 Subject: [PATCH 02/38] kernel-doc: support printing exported and non-exported symbols Currently we use docproc to figure out which symbols are exported, and then docproc calls kernel-doc on specific functions, to get documentation on exported functions. According to git blame and docproc comments, this is due to historical reasons, as functions and their corresponding EXPORT_SYMBOL* may have been in different files. However for more than ten years the recommendation in CodingStyle has been to place the EXPORT_SYMBOL* immediately after the closing function brace line. Additionally, the kernel-doc comments for functions are generally placed above the function definition in the .c files (i.e. where the EXPORT_SYMBOL* is) rather than above the declaration in the .h files. There are some exceptions to this, but AFAICT none of these are included in DocBook documentation using the "!E" docproc directive. Therefore, assuming the EXPORT_SYMBOL* and kernel-doc are with the function definition, kernel-doc can extract the exported vs. not information by making two passes on the input file. Add support for that via the new -export and -internal parameters. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index babb374c043dd..3ad54abe09899 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -59,6 +59,12 @@ Output format selection (mutually exclusive): -text Output plain text format. Output selection (mutually exclusive): + -export Only output documentation for symbols that have been + exported using EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() + in the same FILE. + -internal Only output documentation for symbols that have NOT been + exported using EXPORT_SYMBOL() or EXPORT_SYMBOL_GPL() + in the same FILE. -function NAME Only output documentation for the given function(s) or DOC: section title(s). All other functions and DOC: sections are ignored. May be specified multiple times. @@ -380,6 +386,7 @@ my $doc_block = $doc_com . 'DOC:\s*(.*)?'; my $doc_split_start = '^\s*/\*\*\s*$'; my $doc_split_sect = '\s*\*\s*(@[\w\s]+):(.*)'; my $doc_split_end = '^\s*\*/\s*$'; +my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;'; my %constants; my %parameterdescs; @@ -444,6 +451,12 @@ while ($ARGV[0] =~ m/^-(.*)/) { $function_only = 2; $function = shift @ARGV; $function_table{$function} = 1; + } elsif ($cmd eq "-export") { # only exported symbols + $function_only = 3; + %function_table = () + } elsif ($cmd eq "-internal") { # only non-exported symbols + $function_only = 4; + %function_table = () } elsif ($cmd eq "-v") { $verbose = 1; } elsif (($cmd eq "-h") || ($cmd eq "--help")) { @@ -1971,8 +1984,10 @@ sub output_declaration { my $functype = shift; my $func = "output_${functype}_$output_mode"; if (($function_only==0) || - ( $function_only == 1 && defined($function_table{$name})) || - ( $function_only == 2 && !($functype eq "function" && defined($function_table{$name})))) + ( ($function_only == 1 || $function_only == 3) && + defined($function_table{$name})) || + ( ($function_only == 2 || $function_only == 4) && + !($functype eq "function" && defined($function_table{$name})))) { &$func(@_); $section_counter++; @@ -2675,6 +2690,16 @@ sub process_file($) { return; } + # two passes for -export and -internal + if ($function_only == 3 || $function_only == 4) { + while () { + if (/$export_symbol/o) { + $function_table{$2} = 1; + } + } + seek(IN, 0, 0); + } + $. = 1; $section_counter = 0; From 22cba31bae9dc357622f09d22b82ca5a112b4fde Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 19 May 2016 15:14:05 +0300 Subject: [PATCH 03/38] Documentation/sphinx: add basic working Sphinx configuration and build Add basic configuration and makefile to build documentation from any .rst files under Documentation using Sphinx. For starters, there's just the placeholder index.rst. At the top level Makefile, hook Sphinx documentation targets alongside (but independent of) the DocBook toolchain, having both be run on the various 'make *docs' targets. All Sphinx processing is placed into Documentation/Makefile.sphinx. Both that and the Documentation/DocBook/Makefile are now expected to handle all the documentation targets, explicitly ignoring them if they're not relevant for that particular toolchain. The changes to the existing DocBook Makefile are kept minimal. There is graceful handling of missing Sphinx and rst2pdf (which is needed for pdf output) by checking for the tool and python module, respectively, with informative messages to the user. If the Read the Docs theme (sphinx_rtd_theme) is available, use it, but otherwise gracefully fall back to the Sphinx default theme, with an informative message to the user, and slightly less pretty HTML output. Sphinx can now handle htmldocs, pdfdocs (if rst2pdf is available), epubdocs and xmldocs targets. The output documents are written into per output type subdirectories under Documentation/output. Finally, you can pass options to sphinx-build using the SPHINXBUILD make variable. For example, 'make SPHINXOPTS=-v htmldocs' for more verbose output from Sphinx. This is based on the original work by Jonathan Corbet, but he probably wouldn't recognize this as his own anymore. Signed-off-by: Jani Nikula --- Documentation/DocBook/Makefile | 7 +- Documentation/Makefile.sphinx | 61 ++++++ Documentation/conf.py | 387 +++++++++++++++++++++++++++++++++ Documentation/index.rst | 23 ++ Makefile | 5 +- 5 files changed, 478 insertions(+), 5 deletions(-) create mode 100644 Documentation/Makefile.sphinx create mode 100644 Documentation/conf.py create mode 100644 Documentation/index.rst diff --git a/Documentation/DocBook/Makefile b/Documentation/DocBook/Makefile index d70f9b68174e8..e0c7e1e0590be 100644 --- a/Documentation/DocBook/Makefile +++ b/Documentation/DocBook/Makefile @@ -33,10 +33,6 @@ PDF_METHOD = $(prefer-db2x) PS_METHOD = $(prefer-db2x) -### -# The targets that may be used. -PHONY += xmldocs sgmldocs psdocs pdfdocs htmldocs mandocs installmandocs cleandocs - targets += $(DOCBOOKS) BOOKS := $(addprefix $(obj)/,$(DOCBOOKS)) xmldocs: $(BOOKS) @@ -63,6 +59,9 @@ installmandocs: mandocs sort -k 2 -k 1 | uniq -f 1 | sed -e 's: :/:' | \ xargs install -m 644 -t /usr/local/man/man9/ +# no-op for the DocBook toolchain +epubdocs: + ### #External programs used KERNELDOCXMLREF = $(srctree)/scripts/kernel-doc-xml-ref diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx new file mode 100644 index 0000000000000..25a7970bfe5ce --- /dev/null +++ b/Documentation/Makefile.sphinx @@ -0,0 +1,61 @@ +# -*- makefile -*- +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXBUILD = sphinx-build +SPHINXOPTS = +PAPER = +BUILDDIR = $(obj)/output + +# User-friendly check for sphinx-build +HAVE_SPHINX := $(shell if which $(SPHINXBUILD) >/dev/null 2>&1; then echo 1; else echo 0; fi) + +ifeq ($(HAVE_SPHINX),0) + +.DEFAULT: + $(warning The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed and in PATH, or set the SPHINXBUILD make variable to point to the full path of the '$(SPHINXBUILD)' executable.) + @echo " SKIP Sphinx $@ target." + +else # HAVE_SPHINX + +# User-friendly check for rst2pdf +HAVE_RST2PDF := $(shell if python -c "import rst2pdf" >/dev/null 2>&1; then echo 1; else echo 0; fi) + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/.doctrees $(PAPEROPT_$(PAPER)) -c $(srctree)/$(src) $(SPHINXOPTS) $(srctree)/$(src) +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +quiet_cmd_sphinx = SPHINX $@ + cmd_sphinx = $(SPHINXBUILD) -b $2 $(ALLSPHINXOPTS) $(BUILDDIR)/$2 + +htmldocs: + $(call cmd,sphinx,html) + +pdfdocs: +ifeq ($(HAVE_RST2PDF),0) + $(warning The Python 'rst2pdf' module was not found. Make sure you have the module installed to produce PDF output.) + @echo " SKIP Sphinx $@ target." +else # HAVE_RST2PDF + $(call cmd,sphinx,pdf) +endif # HAVE_RST2PDF + +epubdocs: + $(call cmd,sphinx,epub) + +xmldocs: + $(call cmd,sphinx,xml) + +# no-ops for the Sphinx toolchain +sgmldocs: +psdocs: +mandocs: +installmandocs: + +cleandocs: + $(Q)rm -rf $(BUILDDIR) + +endif # HAVE_SPHINX diff --git a/Documentation/conf.py b/Documentation/conf.py new file mode 100644 index 0000000000000..0f0e20757bb1f --- /dev/null +++ b/Documentation/conf.py @@ -0,0 +1,387 @@ +# -*- coding: utf-8 -*- +# +# The Linux Kernel documentation build configuration file, created by +# sphinx-quickstart on Fri Feb 12 13:51:46 2016. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +#sys.path.insert(0, os.path.abspath('.')) + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [] + +# Gracefully handle missing rst2pdf. +try: + import rst2pdf + extensions += ['rst2pdf.pdfbuilder'] +except ImportError: + pass + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'The Linux Kernel' +copyright = '2016, The kernel development community' +author = 'The kernel development community' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '4.5' +# The full version, including alpha/beta/rc tags. +release = '4.5' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['output'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +#modindex_common_prefix = [] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + +primary_domain = 'C' +highlight_language = 'C' + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. + +# The Read the Docs theme is available from +# - https://github.com/snide/sphinx_rtd_theme +# - https://pypi.python.org/pypi/sphinx_rtd_theme +# - python-sphinx-rtd-theme package (on Debian) +try: + import sphinx_rtd_theme + html_theme = 'sphinx_rtd_theme' + html_theme_path = [sphinx_rtd_theme.get_html_theme_path()] +except ImportError: + sys.stderr.write('Warning: The Sphinx \'sphinx_rtd_theme\' HTML theme was not found. Make sure you have the theme installed to produce pretty HTML output. Falling back to the default theme.\n') + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +#html_theme_options = {} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# " v documentation". +#html_title = None + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +#html_logo = None + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +#html_favicon = None + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +#html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'h', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'r', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'TheLinuxKerneldoc' + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { +# The paper size ('letterpaper' or 'a4paper'). +#'papersize': 'letterpaper', + +# The font size ('10pt', '11pt' or '12pt'). +#'pointsize': '10pt', + +# Additional stuff for the LaTeX preamble. +#'preamble': '', + +# Latex figure (float) alignment +#'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'TheLinuxKernel.tex', 'The Linux Kernel Documentation', + 'The kernel development community', 'manual'), +] + +# The name of an image file (relative to this directory) to place at the top of +# the title page. +#latex_logo = None + +# For "manual" documents, if this is true, then toplevel headings are parts, +# not chapters. +#latex_use_parts = False + +# If true, show page references after internal links. +#latex_show_pagerefs = False + +# If true, show URL addresses after external links. +#latex_show_urls = False + +# Documents to append as an appendix to all manuals. +#latex_appendices = [] + +# If false, no module index is generated. +#latex_domain_indices = True + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'thelinuxkernel', 'The Linux Kernel Documentation', + [author], 1) +] + +# If true, show URL addresses after external links. +#man_show_urls = False + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'TheLinuxKernel', 'The Linux Kernel Documentation', + author, 'TheLinuxKernel', 'One line description of project.', + 'Miscellaneous'), +] + +# Documents to append as an appendix to all manuals. +#texinfo_appendices = [] + +# If false, no module index is generated. +#texinfo_domain_indices = True + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#texinfo_show_urls = 'footnote' + +# If true, do not generate a @detailmenu in the "Top" node's menu. +#texinfo_no_detailmenu = False + + +# -- Options for Epub output ---------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The basename for the epub file. It defaults to the project name. +#epub_basename = project + +# The HTML theme for the epub output. Since the default themes are not +# optimized for small screen space, using the same theme for HTML and epub +# output is usually not wise. This defaults to 'epub', a theme designed to save +# visual space. +#epub_theme = 'epub' + +# The language of the text. It defaults to the language option +# or 'en' if the language is not set. +#epub_language = '' + +# The scheme of the identifier. Typical schemes are ISBN or URL. +#epub_scheme = '' + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +#epub_identifier = '' + +# A unique identification for the text. +#epub_uid = '' + +# A tuple containing the cover image and cover page html template filenames. +#epub_cover = () + +# A sequence of (type, uri, title) tuples for the guide element of content.opf. +#epub_guide = () + +# HTML files that should be inserted before the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_pre_files = [] + +# HTML files that should be inserted after the pages created by sphinx. +# The format is a list of tuples containing the path and title. +#epub_post_files = [] + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + +# The depth of the table of contents in toc.ncx. +#epub_tocdepth = 3 + +# Allow duplicate toc entries. +#epub_tocdup = True + +# Choose between 'default' and 'includehidden'. +#epub_tocscope = 'default' + +# Fix unsupported image types using the Pillow. +#epub_fix_images = False + +# Scale large images. +#epub_max_image_width = 0 + +# How to display URL addresses: 'footnote', 'no', or 'inline'. +#epub_show_urls = 'inline' + +# If false, no index is generated. +#epub_use_index = True + +#======= +# rst2pdf +# +# Grouping the document tree into PDF files. List of tuples +# (source start file, target name, title, author, options). +# +# See the Sphinx chapter of http://ralsina.me/static/manual.pdf +# +# FIXME: Do not add the index file here; the result will be too big. Adding +# multiple PDF files here actually tries to get the cross-referencing right +# *between* PDF files. +pdf_documents = [ + ('index', u'Kernel', u'Kernel', u'J. Random Bozo'), +] diff --git a/Documentation/index.rst b/Documentation/index.rst new file mode 100644 index 0000000000000..71a276f34c7f0 --- /dev/null +++ b/Documentation/index.rst @@ -0,0 +1,23 @@ +.. The Linux Kernel documentation master file, created by + sphinx-quickstart on Fri Feb 12 13:51:46 2016. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to The Linux Kernel's documentation! +============================================ + +Nothing for you to see here *yet*. Please move along. + +Contents: + +.. toctree:: + :maxdepth: 2 + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + diff --git a/Makefile b/Makefile index 0f70de63cfdb4..8908a51b895a3 100644 --- a/Makefile +++ b/Makefile @@ -1412,8 +1412,11 @@ $(help-board-dirs): help-%: # Documentation targets # --------------------------------------------------------------------------- -%docs: scripts_basic FORCE +DOC_TARGETS := xmldocs sgmldocs psdocs pdfdocs htmldocs mandocs installmandocs epubdocs cleandocs +PHONY += $(DOC_TARGETS) +$(DOC_TARGETS): scripts_basic FORCE $(Q)$(MAKE) $(build)=scripts build_docproc build_check-lc_ctype + $(Q)$(MAKE) $(build)=Documentation -f $(srctree)/Documentation/Makefile.sphinx $@ $(Q)$(MAKE) $(build)=Documentation/DocBook $@ else # KBUILD_EXTMOD From 81cd318102aedee64d3bd38d2e04ea333569abfd Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 19 May 2016 20:53:23 +0300 Subject: [PATCH 04/38] Documentation: add .gitignore The Sphinx output directory is generated. Signed-off-by: Jani Nikula --- Documentation/.gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 Documentation/.gitignore diff --git a/Documentation/.gitignore b/Documentation/.gitignore new file mode 100644 index 0000000000000..53752db253e3b --- /dev/null +++ b/Documentation/.gitignore @@ -0,0 +1 @@ +output From c56de1db54dfbbdfc02a500388952af16e3a9368 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Wed, 18 May 2016 23:30:30 +0300 Subject: [PATCH 05/38] Documentation/sphinx: add Sphinx kernel-doc directive extension Add an extension to handle kernel-doc directives, to call kernel-doc according to the arguments and parameters given to the reStructuredText directive. The syntax for the kernel-doc directive is: .. kernel-doc:: FILENAME :export: :internal: :functions: FUNCTION [FUNCTION ...] :doc: SECTION TITLE Of the directive options export, internal, functions, and doc, currently only one option may be given at a time. The FILENAME is relative from the kernel source tree root. The extension notifies Sphinx about the document dependency on FILENAME, causing the document to be rebuilt when the file has been changed. Signed-off-by: Jani Nikula --- Documentation/sphinx/kernel-doc.py | 102 +++++++++++++++++++++++++++++ 1 file changed, 102 insertions(+) create mode 100644 Documentation/sphinx/kernel-doc.py diff --git a/Documentation/sphinx/kernel-doc.py b/Documentation/sphinx/kernel-doc.py new file mode 100644 index 0000000000000..87a1332fe934f --- /dev/null +++ b/Documentation/sphinx/kernel-doc.py @@ -0,0 +1,102 @@ +# coding=utf-8 +# +# Copyright © 2016 Intel Corporation +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice (including the next +# paragraph) shall be included in all copies or substantial portions of the +# Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# Authors: +# Jani Nikula + +import os +import subprocess +import sys + +from docutils import nodes, statemachine +from docutils.parsers.rst import directives +from sphinx.util.compat import Directive + +class KernelDocDirective(Directive): + """Extract kernel-doc comments from the specified file""" + required_argument = 1 + optional_arguments = 4 + option_spec = { + 'doc': directives.unchanged_required, + 'functions': directives.unchanged_required, + 'export': directives.flag, + 'internal': directives.flag, + } + has_content = False + + def run(self): + env = self.state.document.settings.env + cmd = [env.config.kerneldoc_bin, '-rst'] + + filename = env.config.kerneldoc_srctree + '/' + self.arguments[0] + + # Tell sphinx of the dependency + env.note_dependency(os.path.abspath(filename)) + + tab_width = self.options.get('tab-width', self.state.document.settings.tab_width) + source = self.state_machine.input_lines.source(self.lineno - self.state_machine.input_offset - 1) + + # FIXME: make this nicer and more robust against errors + if 'export' in self.options: + cmd += ['-export'] + elif 'internal' in self.options: + cmd += ['-internal'] + elif 'doc' in self.options: + cmd += ['-function', str(self.options.get('doc'))] + elif 'functions' in self.options: + for f in str(self.options.get('functions')).split(' '): + cmd += ['-function', f] + + cmd += [filename] + + try: + env.app.verbose('calling kernel-doc \'%s\'' % (" ".join(cmd))) + + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) + out, err = p.communicate() + + # assume the kernel sources are utf-8 + out, err = unicode(out, 'utf-8'), unicode(err, 'utf-8') + + if p.returncode != 0: + sys.stderr.write(err) + + env.app.warn('kernel-doc \'%s\' failed with return code %d' % (" ".join(cmd), p.returncode)) + return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] + elif env.config.kerneldoc_verbosity > 0: + sys.stderr.write(err) + + lines = statemachine.string2lines(out, tab_width, convert_whitespace=True) + self.state_machine.insert_input(lines, source) + return [] + except Exception as e: + env.app.warn('kernel-doc \'%s\' processing failed with: %s' % + (" ".join(cmd), str(e))) + return [nodes.error(None, nodes.paragraph(text = "kernel-doc missing"))] + +def setup(app): + app.add_config_value('kerneldoc_bin', None, 'env') + app.add_config_value('kerneldoc_srctree', None, 'env') + app.add_config_value('kerneldoc_verbosity', 1, 'env') + + app.add_directive('kernel-doc', KernelDocDirective) From 24dcdeb28b55afafa9cdf6acedd78cf1aa8f8428 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Fri, 20 May 2016 11:51:47 +0300 Subject: [PATCH 06/38] Documentation/sphinx: configure the kernel-doc extension Tell Sphinx where to find the extension, and pass on the kernel src tree and kernel-doc paths to the extension. With this, any .rst files under Documentation may contain the kernel-doc rst directive to include kernel-doc documentation from any source file. While building, it may be handy to pass kernel-doc extension configuration on the command line. For example, 'make SPHINXOPTS="-D kerneldoc_verbosity=0" htmldocs' silences all stderr output from kernel-doc when the kernel-doc exit code is 0. (The stderr will be logged unconditionally when the exit code is non-zero.) Signed-off-by: Jani Nikula --- Documentation/Makefile.sphinx | 4 +++- Documentation/conf.py | 10 ++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index 25a7970bfe5ce..8a662be9bd871 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -25,7 +25,9 @@ HAVE_RST2PDF := $(shell if python -c "import rst2pdf" >/dev/null 2>&1; then echo # Internal variables. PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter -ALLSPHINXOPTS = -d $(BUILDDIR)/.doctrees $(PAPEROPT_$(PAPER)) -c $(srctree)/$(src) $(SPHINXOPTS) $(srctree)/$(src) +KERNELDOC = $(srctree)/scripts/kernel-doc +KERNELDOC_CONF = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC) +ALLSPHINXOPTS = -d $(BUILDDIR)/.doctrees $(KERNELDOC_CONF) $(PAPEROPT_$(PAPER)) -c $(srctree)/$(src) $(SPHINXOPTS) $(srctree)/$(src) # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . diff --git a/Documentation/conf.py b/Documentation/conf.py index 0f0e20757bb1f..8b72577f38f55 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -18,7 +18,7 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +sys.path.insert(0, os.path.abspath('sphinx')) # -- General configuration ------------------------------------------------ @@ -28,7 +28,7 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = [] +extensions = ['kernel-doc'] # Gracefully handle missing rst2pdf. try: @@ -385,3 +385,9 @@ pdf_documents = [ ('index', u'Kernel', u'Kernel', u'J. Random Bozo'), ] + +# kernel-doc extension configuration for running Sphinx directly (e.g. by Read +# the Docs). In a normal build, these are supplied from the Makefile via command +# line arguments. +kerneldoc_bin = '../scripts/kernel-doc' +kerneldoc_srctree = '..' From c13ce448c8532965eb4e1115e78cb5b8ff8261a3 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sat, 28 May 2016 15:25:41 +0300 Subject: [PATCH 07/38] Documentation/sphinx: set version and release properly Read the version and release from the top level Makefile (for use when Sphinx is invoked directly, by e.g. Read the Docs), but override them via Sphinx command line arguments in a normal documentation build. Signed-off-by: Jani Nikula --- Documentation/Makefile.sphinx | 2 +- Documentation/conf.py | 29 +++++++++++++++++++++++++---- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/Documentation/Makefile.sphinx b/Documentation/Makefile.sphinx index 8a662be9bd871..addf32309bc35 100644 --- a/Documentation/Makefile.sphinx +++ b/Documentation/Makefile.sphinx @@ -27,7 +27,7 @@ PAPEROPT_a4 = -D latex_paper_size=a4 PAPEROPT_letter = -D latex_paper_size=letter KERNELDOC = $(srctree)/scripts/kernel-doc KERNELDOC_CONF = -D kerneldoc_srctree=$(srctree) -D kerneldoc_bin=$(KERNELDOC) -ALLSPHINXOPTS = -d $(BUILDDIR)/.doctrees $(KERNELDOC_CONF) $(PAPEROPT_$(PAPER)) -c $(srctree)/$(src) $(SPHINXOPTS) $(srctree)/$(src) +ALLSPHINXOPTS = -D version=$(KERNELVERSION) -D release=$(KERNELRELEASE) -d $(BUILDDIR)/.doctrees $(KERNELDOC_CONF) $(PAPEROPT_$(PAPER)) -c $(srctree)/$(src) $(SPHINXOPTS) $(srctree)/$(src) # the i18n builder cannot share the environment and doctrees with the others I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . diff --git a/Documentation/conf.py b/Documentation/conf.py index 8b72577f38f55..6cc41a0555a3f 100644 --- a/Documentation/conf.py +++ b/Documentation/conf.py @@ -60,10 +60,31 @@ # |version| and |release|, also used in various other places throughout the # built documents. # -# The short X.Y version. -version = '4.5' -# The full version, including alpha/beta/rc tags. -release = '4.5' +# In a normal build, version and release are are set to KERNELVERSION and +# KERNELRELEASE, respectively, from the Makefile via Sphinx command line +# arguments. +# +# The following code tries to extract the information by reading the Makefile, +# when Sphinx is run directly (e.g. by Read the Docs). +try: + makefile_version = None + makefile_patchlevel = None + for line in open('../Makefile'): + key, val = [x.strip() for x in line.split('=', 2)] + if key == 'VERSION': + makefile_version = val + elif key == 'PATCHLEVEL': + makefile_patchlevel = val + if makefile_version and makefile_patchlevel: + break +except: + pass +finally: + if makefile_version and makefile_patchlevel: + version = release = makefile_version + '.' + makefile_patchlevel + else: + sys.stderr.write('Warning: Could not extract kernel version\n') + version = release = "unknown version" # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. From 89a66d76109935aad387a5058f120aeb8246ae40 Mon Sep 17 00:00:00 2001 From: Jonathan Corbet Date: Sun, 6 Mar 2016 20:29:31 -0700 Subject: [PATCH 08/38] sphinx: cheesy script to convert .tmpl files This script uses pandoc to convert existing DocBook template files to RST templates. A couple of sed scripts are need to massage things both before and after the conversion, but the result is then usable with no hand editing. [Jani: Change usage to tmplcvt . Fix escaping for docproc directives. Add support the new kernel-doc extension.] Signed-off-by: Jonathan Corbet Signed-off-by: Jani Nikula --- Documentation/sphinx/convert_template.sed | 14 ++++++++++++++ Documentation/sphinx/post_convert.sed | 19 +++++++++++++++++++ Documentation/sphinx/tmplcvt | 19 +++++++++++++++++++ 3 files changed, 52 insertions(+) create mode 100644 Documentation/sphinx/convert_template.sed create mode 100644 Documentation/sphinx/post_convert.sed create mode 100755 Documentation/sphinx/tmplcvt diff --git a/Documentation/sphinx/convert_template.sed b/Documentation/sphinx/convert_template.sed new file mode 100644 index 0000000000000..d53bb8220a264 --- /dev/null +++ b/Documentation/sphinx/convert_template.sed @@ -0,0 +1,14 @@ +# +# Pandoc doesn't grok or , so convert them +# ahead of time. +# +# Use "$bq" instead of "`" so that pandoc won't mess with it. +# +s%\([^<(]\+\)()%:c:func:$bq\1$bq%g +s%\([^<(]\+\)%:c:func:$bq\1$bq%g +s%struct *\([^<]\+\)%:ref:$bqstruct \1$bq%g +s%\([^<]\+\)%:ref:$bqstruct \1$bq%g +# +# Wrap docproc directives in para and code blocks. +# +s%^\(!.*\)$%DOCPROC: \1% diff --git a/Documentation/sphinx/post_convert.sed b/Documentation/sphinx/post_convert.sed new file mode 100644 index 0000000000000..ce17049f6a8ee --- /dev/null +++ b/Documentation/sphinx/post_convert.sed @@ -0,0 +1,19 @@ +# +# pandoc thinks that both "_" needs to be escaped. Remove the extra +# backslashes. Also put in proper backquotes now that pandoc won't quote +# them. +# +s/$bq/`/g +s/\\_/_/g +# +# Unwrap docproc directives. +# +s/^``DOCPROC: !E\(.*\)``$/.. kernel-doc:: \1\n :export:/ +s/^``DOCPROC: !I\(.*\)``$/.. kernel-doc:: \1\n :internal:/ +s/^``DOCPROC: !F\([^ ]*\) \(.*\)``$/.. kernel-doc:: \1\n :functions: \2/ +s/^``DOCPROC: !P\([^ ]*\) \(.*\)``$/.. kernel-doc:: \1\n :doc: \2/ +s/^``DOCPROC: \(!.*\)``$/.. WARNING: DOCPROC directive not supported: \1/ +# +# Trim trailing whitespace. +# +s/[[:space:]]*$// diff --git a/Documentation/sphinx/tmplcvt b/Documentation/sphinx/tmplcvt new file mode 100755 index 0000000000000..909a73065e0a5 --- /dev/null +++ b/Documentation/sphinx/tmplcvt @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Convert a template file into something like RST +# +# fix +# feed to pandoc +# fix \_ +# title line? +# + +in=$1 +rst=$2 +tmp=$rst.tmp + +cp $in $tmp +sed --in-place -f convert_template.sed $tmp +pandoc -s -S -f docbook -t rst -o $rst $tmp +sed --in-place -f post_convert.sed $rst +rm $tmp From 2e83ecb834ca923806fd58fab17d9f405767e1d9 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 10:15:18 +0300 Subject: [PATCH 09/38] sphinx: update docbook->rst conversion script match C domain spec Function references should include the parens (), struct references should not include "struct". Signed-off-by: Jani Nikula --- Documentation/sphinx/convert_template.sed | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Documentation/sphinx/convert_template.sed b/Documentation/sphinx/convert_template.sed index d53bb8220a264..0553eabeac793 100644 --- a/Documentation/sphinx/convert_template.sed +++ b/Documentation/sphinx/convert_template.sed @@ -4,10 +4,10 @@ # # Use "$bq" instead of "`" so that pandoc won't mess with it. # -s%\([^<(]\+\)()%:c:func:$bq\1$bq%g -s%\([^<(]\+\)%:c:func:$bq\1$bq%g -s%struct *\([^<]\+\)%:ref:$bqstruct \1$bq%g -s%\([^<]\+\)%:ref:$bqstruct \1$bq%g +s%\([^<(]\+\)()%:c:func:$bq\1()$bq%g +s%\([^<(]\+\)%:c:func:$bq\1()$bq%g +s%struct *\([^<]\+\)%:c:type:$bq\1$bq%g +s%\([^<]\+\)%:c:type:$bq\1$bq%g # # Wrap docproc directives in para and code blocks. # From 30ca7aaf279944da930b587b05d9b325ec7f821d Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 10:40:17 +0300 Subject: [PATCH 10/38] Documentation/sphinx: nicer referencing of struct in docbook->rst conversion Add "struct" in the label of the reference. Signed-off-by: Jani Nikula --- Documentation/sphinx/convert_template.sed | 10 +++++++--- Documentation/sphinx/post_convert.sed | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Documentation/sphinx/convert_template.sed b/Documentation/sphinx/convert_template.sed index 0553eabeac793..c1503fcca4eca 100644 --- a/Documentation/sphinx/convert_template.sed +++ b/Documentation/sphinx/convert_template.sed @@ -2,12 +2,16 @@ # Pandoc doesn't grok or , so convert them # ahead of time. # -# Use "$bq" instead of "`" so that pandoc won't mess with it. +# Use the following escapes to pass through pandoc: +# $bq = "`" +# $lt = "<" +# $gt = ">" # s%\([^<(]\+\)()%:c:func:$bq\1()$bq%g s%\([^<(]\+\)%:c:func:$bq\1()$bq%g -s%struct *\([^<]\+\)%:c:type:$bq\1$bq%g -s%\([^<]\+\)%:c:type:$bq\1$bq%g +s%struct *\([^<]\+\)%:c:type:$bqstruct \1 $lt\1$gt$bq%g +s%struct \([^<]\+\)%:c:type:$bqstruct \1 $lt\1$gt$bq%g +s%\([^<]\+\)%:c:type:$bqstruct \1 $lt\1$gt$bq%g # # Wrap docproc directives in para and code blocks. # diff --git a/Documentation/sphinx/post_convert.sed b/Documentation/sphinx/post_convert.sed index ce17049f6a8ee..392770bac53b7 100644 --- a/Documentation/sphinx/post_convert.sed +++ b/Documentation/sphinx/post_convert.sed @@ -1,9 +1,13 @@ # -# pandoc thinks that both "_" needs to be escaped. Remove the extra -# backslashes. Also put in proper backquotes now that pandoc won't quote -# them. +# Unescape. # s/$bq/`/g +s/$lt//g +# +# pandoc thinks that both "_" needs to be escaped. Remove the extra +# backslashes. +# s/\\_/_/g # # Unwrap docproc directives. From 48af606ad8912f90e1539621a26e86672976d8ae Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 14:56:05 +0300 Subject: [PATCH 11/38] kernel-doc: add names for states and substates Make the state machine a bit more readable by adding constants for parser states and inline member documentation parser substates. While at it, rename the "split" documentation to "inline" documentation. No functional changes. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 91 ++++++++++++++++++++++++---------------------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 3ad54abe09899..cb5fd248ac57e 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -350,24 +350,29 @@ my $section_counter = 0; my $lineprefix=""; -# states -# 0 - normal code -# 1 - looking for function name -# 2 - scanning field start. -# 3 - scanning prototype. -# 4 - documentation block -# 5 - gathering documentation outside main block +# Parser states +use constant { + STATE_NORMAL => 0, # normal code + STATE_NAME => 1, # looking for function name + STATE_FIELD => 2, # scanning field start + STATE_PROTO => 3, # scanning prototype + STATE_DOCBLOCK => 4, # documentation block + STATE_INLINE => 5, # gathering documentation outside main block +}; my $state; my $in_doc_sect; -# Split Doc State -# 0 - Invalid (Before start or after finish) -# 1 - Is started (the /** was found inside a struct) -# 2 - The @parameter header was found, start accepting multi paragraph text. -# 3 - Finished (the */ was found) -# 4 - Error - Comment without header was found. Spit a warning as it's not -# proper kernel-doc and ignore the rest. -my $split_doc_state; +# Inline documentation state +use constant { + STATE_INLINE_NA => 0, # not applicable ($state != STATE_INLINE) + STATE_INLINE_NAME => 1, # looking for member name (@foo:) + STATE_INLINE_TEXT => 2, # looking for member documentation + STATE_INLINE_END => 3, # done + STATE_INLINE_ERROR => 4, # error - Comment without header was found. + # Spit a warning as it's not + # proper kernel-doc and ignore the rest. +}; +my $inline_doc_state; #declaration types: can be # 'function', 'struct', 'union', 'enum', 'typedef' @@ -383,9 +388,9 @@ my $doc_decl = $doc_com . '(\w+)'; my $doc_sect = $doc_com . '([' . $doc_special . ']?[\w\s]+):(.*)'; my $doc_content = $doc_com_body . '(.*)'; my $doc_block = $doc_com . 'DOC:\s*(.*)?'; -my $doc_split_start = '^\s*/\*\*\s*$'; -my $doc_split_sect = '\s*\*\s*(@[\w\s]+):(.*)'; -my $doc_split_end = '^\s*\*/\s*$'; +my $doc_inline_start = '^\s*/\*\*\s*$'; +my $doc_inline_sect = '\s*\*\s*(@[\w\s]+):(.*)'; +my $doc_inline_end = '^\s*\*/\s*$'; my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;'; my %constants; @@ -2497,8 +2502,8 @@ sub reset_state { $struct_actual = ""; $prototype = ""; - $state = 0; - $split_doc_state = 0; + $state = STATE_NORMAL; + $inline_doc_state = STATE_INLINE_NA; } sub tracepoint_munge($) { @@ -2707,14 +2712,14 @@ sub process_file($) { while (s/\\\s*$//) { $_ .= ; } - if ($state == 0) { + if ($state == STATE_NORMAL) { if (/$doc_start/o) { - $state = 1; # next line is always the function name + $state = STATE_NAME; # next line is always the function name $in_doc_sect = 0; } - } elsif ($state == 1) { # this line is the function name (always) + } elsif ($state == STATE_NAME) {# this line is the function name (always) if (/$doc_block/o) { - $state = 4; + $state = STATE_DOCBLOCK; $contents = ""; if ( $1 eq "" ) { $section = $section_intro; @@ -2728,7 +2733,7 @@ sub process_file($) { $identifier = $1; } - $state = 2; + $state = STATE_FIELD; if (/-(.*)/) { # strip leading/trailing/multiple spaces $descr= $1; @@ -2766,9 +2771,9 @@ sub process_file($) { print STDERR "${file}:$.: warning: Cannot understand $_ on line $.", " - I thought it was a doc line\n"; ++$warnings; - $state = 0; + $state = STATE_NORMAL; } - } elsif ($state == 2) { # look for head: lines, and include content + } elsif ($state == STATE_FIELD) { # look for head: lines, and include content if (/$doc_sect/o) { $newsection = $1; $newcontents = $2; @@ -2806,7 +2811,7 @@ sub process_file($) { } $prototype = ""; - $state = 3; + $state = STATE_PROTO; $brcount = 0; # print STDERR "end of doc comment, looking for prototype\n"; } elsif (/$doc_content/) { @@ -2834,9 +2839,9 @@ sub process_file($) { print STDERR "${file}:$.: warning: bad line: $_"; ++$warnings; } - } elsif ($state == 5) { # scanning for split parameters + } elsif ($state == STATE_INLINE) { # scanning for inline parameters # First line (state 1) needs to be a @parameter - if ($split_doc_state == 1 && /$doc_split_sect/o) { + if ($inline_doc_state == STATE_INLINE_NAME && /$doc_inline_sect/o) { $section = $1; $contents = $2; if ($contents ne "") { @@ -2846,37 +2851,37 @@ sub process_file($) { } $contents .= "\n"; } - $split_doc_state = 2; + $inline_doc_state = STATE_INLINE_TEXT; # Documentation block end */ - } elsif (/$doc_split_end/) { + } elsif (/$doc_inline_end/) { if (($contents ne "") && ($contents ne "\n")) { dump_section($file, $section, xml_escape($contents)); $section = $section_default; $contents = ""; } - $state = 3; - $split_doc_state = 0; + $state = STATE_PROTO; + $inline_doc_state = STATE_INLINE_NA; # Regular text } elsif (/$doc_content/) { - if ($split_doc_state == 2) { + if ($inline_doc_state == STATE_INLINE_TEXT) { $contents .= $1 . "\n"; - } elsif ($split_doc_state == 1) { - $split_doc_state = 4; + } elsif ($inline_doc_state == STATE_INLINE_NAME) { + $inline_doc_state = STATE_INLINE_ERROR; print STDERR "Warning(${file}:$.): "; print STDERR "Incorrect use of kernel-doc format: $_"; ++$warnings; } } - } elsif ($state == 3) { # scanning for function '{' (end of prototype) - if (/$doc_split_start/) { - $state = 5; - $split_doc_state = 1; + } elsif ($state == STATE_PROTO) { # scanning for function '{' (end of prototype) + if (/$doc_inline_start/) { + $state = STATE_INLINE; + $inline_doc_state = STATE_INLINE_NAME; } elsif ($decl_type eq 'function') { process_state3_function($_, $file); } else { process_state3_type($_, $file); } - } elsif ($state == 4) { + } elsif ($state == STATE_DOCBLOCK) { # Documentation block if (/$doc_block/) { dump_doc_section($file, $section, xml_escape($contents)); @@ -2907,7 +2912,7 @@ sub process_file($) { %sections = (); @sectionlist = (); $prototype = ""; - $state = 0; + $state = STATE_NORMAL; } elsif (/$doc_content/) { From b6c3f456cfed53e9f06f431270c9dcd52010785e Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sun, 29 May 2016 22:19:35 +0300 Subject: [PATCH 12/38] kernel-doc: add names for output selection Make the output selection a bit more readable by adding constants for the various types of output selection. While at it, actually call the variable for choosing what to output $output_selection. No functional changes. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 47 +++++++++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index cb5fd248ac57e..dd08944b0a6f7 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -312,7 +312,15 @@ my $no_doc_sections = 0; my @highlights = @highlights_man; my $blankline = $blankline_man; my $modulename = "Kernel API"; -my $function_only = 0; + +use constant { + OUTPUT_ALL => 0, # output all symbols and doc sections + OUTPUT_INCLUDE => 1, # output only specified symbols + OUTPUT_EXCLUDE => 2, # output everything except specified symbols + OUTPUT_EXPORTED => 3, # output exported symbols + OUTPUT_INTERNAL => 4, # output non-exported symbols +}; +my $output_selection = OUTPUT_ALL; my $show_not_found = 0; my @build_time; @@ -449,18 +457,18 @@ while ($ARGV[0] =~ m/^-(.*)/) { } elsif ($cmd eq "-module") { # not needed for XML, inherits from calling document $modulename = shift @ARGV; } elsif ($cmd eq "-function") { # to only output specific functions - $function_only = 1; + $output_selection = OUTPUT_INCLUDE; $function = shift @ARGV; $function_table{$function} = 1; - } elsif ($cmd eq "-nofunction") { # to only output specific functions - $function_only = 2; + } elsif ($cmd eq "-nofunction") { # output all except specific functions + $output_selection = OUTPUT_EXCLUDE; $function = shift @ARGV; $function_table{$function} = 1; } elsif ($cmd eq "-export") { # only exported symbols - $function_only = 3; + $output_selection = OUTPUT_EXPORTED; %function_table = () } elsif ($cmd eq "-internal") { # only non-exported symbols - $function_only = 4; + $output_selection = OUTPUT_INTERNAL; %function_table = () } elsif ($cmd eq "-v") { $verbose = 1; @@ -530,15 +538,17 @@ sub dump_doc_section { return; } - if (($function_only == 0) || - ( $function_only == 1 && defined($function_table{$name})) || - ( $function_only == 2 && !defined($function_table{$name}))) + if (($output_selection == OUTPUT_ALL) || + ($output_selection == OUTPUT_INCLUDE && + defined($function_table{$name})) || + ($output_selection == OUTPUT_EXCLUDE && + !defined($function_table{$name}))) { dump_section($file, $name, $contents); output_blockhead({'sectionlist' => \@sectionlist, 'sections' => \%sections, 'module' => $modulename, - 'content-only' => ($function_only != 0), }); + 'content-only' => ($output_selection != OUTPUT_ALL), }); } } @@ -1988,11 +1998,13 @@ sub output_declaration { my $name = shift; my $functype = shift; my $func = "output_${functype}_$output_mode"; - if (($function_only==0) || - ( ($function_only == 1 || $function_only == 3) && - defined($function_table{$name})) || - ( ($function_only == 2 || $function_only == 4) && - !($functype eq "function" && defined($function_table{$name})))) + if (($output_selection == OUTPUT_ALL) || + (($output_selection == OUTPUT_INCLUDE || + $output_selection == OUTPUT_EXPORTED) && + defined($function_table{$name})) || + (($output_selection == OUTPUT_EXCLUDE || + $output_selection == OUTPUT_INTERNAL) && + !($functype eq "function" && defined($function_table{$name})))) { &$func(@_); $section_counter++; @@ -2696,7 +2708,8 @@ sub process_file($) { } # two passes for -export and -internal - if ($function_only == 3 || $function_only == 4) { + if ($output_selection == OUTPUT_EXPORTED || + $output_selection == OUTPUT_INTERNAL) { while () { if (/$export_symbol/o) { $function_table{$2} = 1; @@ -2929,7 +2942,7 @@ sub process_file($) { } if ($initial_section_counter == $section_counter) { print STDERR "${file}:1: warning: no structured comments found\n"; - if (($function_only == 1) && ($show_not_found == 1)) { + if (($output_selection == OUTPUT_INCLUDE) && ($show_not_found == 1)) { print STDERR " Was looking for '$_'.\n" for keys %function_table; } if ($output_mode eq "xml") { From 9e72184b55df2b5b8ebdcf0470bd43ef32dc518f Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sun, 29 May 2016 22:27:35 +0300 Subject: [PATCH 13/38] kernel-doc/rst: do not output DOC: section titles for requested ones If the user requests a specific DOC: section by name, do not output its section title. In these cases, the surrounding context already has a heading, and the DOC: section title is only used as an identifier and a heading for clarity in the source file. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index dd08944b0a6f7..659d529b99d85 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1764,7 +1764,9 @@ sub output_blockhead_rst(%) { my ($parameter, $section); foreach $section (@{$args{'sectionlist'}}) { - print "**$section**\n\n"; + if ($output_selection != OUTPUT_INCLUDE) { + print "**$section**\n\n"; + } output_highlight_rst($args{'sections'}{$section}); print "\n"; } From a19bce6433c2566dd210f4245525c297ca952574 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 11:28:16 +0300 Subject: [PATCH 14/38] kernel-doc/rst: reference functions according to C domain spec The Sphinx C domain spec says function references should include the parens (). Signed-off-by: Jani Nikula --- scripts/kernel-doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 659d529b99d85..e8651d7cf1cd6 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -280,7 +280,7 @@ my $blankline_text = ""; # rst-mode my @highlights_rst = ( [$type_constant, "``\$1``"], - [$type_func, "\\:c\\:func\\:`\$1`"], + [$type_func, "\\:c\\:func\\:`\$1()`"], [$type_struct_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], [$type_enum_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], [$type_struct, "\\:c\\:type\\:`struct \$1 <\$1>`"], From a7291e7e03f8b45a4b028a410063dc94f9bff8c0 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 13:57:06 +0300 Subject: [PATCH 15/38] kernel-doc/rst: &foo references are more universal than structs It's possible to use &foo to reference structs, enums, typedefs, etc. in the Sphinx C domain. Thus do not prefix the links with "struct". Signed-off-by: Jani Nikula --- scripts/kernel-doc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index e8651d7cf1cd6..e7aa792e7f1b2 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -283,7 +283,8 @@ my @highlights_rst = ( [$type_func, "\\:c\\:func\\:`\$1()`"], [$type_struct_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], [$type_enum_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], - [$type_struct, "\\:c\\:type\\:`struct \$1 <\$1>`"], + # in rst this can refer to any type + [$type_struct, "\\:c\\:type\\:`\$1`"], [$type_param, "**\$1**"] ); my $blankline_rst = "\n"; From 47ae7aed34ee9017e9eeb2ad066786239456a90f Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 13:57:18 +0300 Subject: [PATCH 16/38] kernel-doc/rst: add support for &union foo and &typedef foo references Let the user use "&union foo" and "&typedef foo" to reference foo. The difference to using "union &foo", "typedef &foo", or just "&foo" (which are valid too) is that "union" and "typedef" become part of the link text. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index e7aa792e7f1b2..446c0912395e1 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -212,6 +212,8 @@ my $type_struct_xml = '\\&((struct\s*)*[_\w]+)'; my $type_env = '(\$\w+)'; my $type_enum_full = '\&(enum)\s*([_\w]+)'; my $type_struct_full = '\&(struct)\s*([_\w]+)'; +my $type_typedef_full = '\&(typedef)\s*([_\w]+)'; +my $type_union_full = '\&(union)\s*([_\w]+)'; # Output conversion substitutions. # One for each output format @@ -283,6 +285,8 @@ my @highlights_rst = ( [$type_func, "\\:c\\:func\\:`\$1()`"], [$type_struct_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], [$type_enum_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], + [$type_typedef_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], + [$type_union_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], # in rst this can refer to any type [$type_struct, "\\:c\\:type\\:`\$1`"], [$type_param, "**\$1**"] From f3341dcf3bdcd1209b2911f35e4e970b789c4744 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 16:35:02 +0300 Subject: [PATCH 17/38] kernel-doc/rst: add support for struct/union/enum member references Link "&foo->bar", "&foo->bar()", "&foo.bar", and "&foo.bar()" to the struct/union/enum foo definition. The members themselves do not currently have anchors to link to, but this is better than nothing, and promotes a universal notation. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 446c0912395e1..e0fd14f6d711a 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -214,6 +214,8 @@ my $type_enum_full = '\&(enum)\s*([_\w]+)'; my $type_struct_full = '\&(struct)\s*([_\w]+)'; my $type_typedef_full = '\&(typedef)\s*([_\w]+)'; my $type_union_full = '\&(union)\s*([_\w]+)'; +my $type_member = '\&([_\w]+)((\.|->)[_\w]+)'; +my $type_member_func = $type_member . '\(\)'; # Output conversion substitutions. # One for each output format @@ -282,6 +284,9 @@ my $blankline_text = ""; # rst-mode my @highlights_rst = ( [$type_constant, "``\$1``"], + # Note: need to escape () to avoid func matching later + [$type_member_func, "\\:c\\:type\\:`\$1\$2\\\\(\\\\) <\$1>`"], + [$type_member, "\\:c\\:type\\:`\$1\$2 <\$1>`"], [$type_func, "\\:c\\:func\\:`\$1()`"], [$type_struct_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], [$type_enum_full, "\\:c\\:type\\:`\$1 \$2 <\$2>`"], From 9c9193c49c1f1662b00476b3d0697a1be37c6b08 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 16:56:27 +0300 Subject: [PATCH 18/38] kernel-doc/rst: drop redundant unescape in highlighting This bit is already done by xml_unescape() above. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index e0fd14f6d711a..8f9eac5093775 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1796,7 +1796,6 @@ sub output_highlight_rst { if ($line eq "") { print $lineprefix, $blankline; } else { - $line =~ s/\\\\\\/\&/g; print $lineprefix, $line; } print "\n"; From c099ff6989baf286da8eaed5c7b3d18ae60ea2e7 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 17:18:17 +0300 Subject: [PATCH 19/38] kernel-doc/rst: highlight function/struct/enum purpose lines too Let the user use @foo, &bar, %baz, etc. in the first kernel-doc purpose line too. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 8f9eac5093775..76bad55c031ec 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1805,6 +1805,7 @@ sub output_highlight_rst { sub output_function_rst(%) { my %args = %{$_[0]}; my ($parameter, $section); + my $oldprefix = $lineprefix; my $start; print ".. c:function:: "; @@ -1829,9 +1830,13 @@ sub output_function_rst(%) { print $type . " " . $parameter; } } - print ")\n\n " . $args{'purpose'} . "\n\n"; + print ")\n\n"; + $lineprefix = " "; + output_highlight_rst($args{'purpose'}); + print "\n"; print ":Parameters:\n\n"; + $lineprefix = " "; foreach $parameter (@{$args{'parameterlist'}}) { my $parameter_name = $parameter; #$parameter_name =~ s/\[.*//; @@ -1844,15 +1849,14 @@ sub output_function_rst(%) { } if (defined($args{'parameterdescs'}{$parameter_name}) && $args{'parameterdescs'}{$parameter_name} ne $undescribed) { - my $oldprefix = $lineprefix; - $lineprefix = " "; output_highlight_rst($args{'parameterdescs'}{$parameter_name}); - $lineprefix = $oldprefix; } else { print "\n _undescribed_\n"; } print "\n"; } + + $lineprefix = $oldprefix; output_section_rst(@_); } @@ -1874,14 +1878,16 @@ sub output_section_rst(%) { sub output_enum_rst(%) { my %args = %{$_[0]}; my ($parameter); + my $oldprefix = $lineprefix; my $count; my $name = "enum " . $args{'enum'}; print "\n\n.. c:type:: " . $name . "\n\n"; - print " " . $args{'purpose'} . "\n\n"; + $lineprefix = " "; + output_highlight_rst($args{'purpose'}); + print "\n"; print "..\n\n:Constants:\n\n"; - my $oldprefix = $lineprefix; $lineprefix = " "; foreach $parameter (@{$args{'parameterlist'}}) { print " `$parameter`\n"; @@ -1892,6 +1898,7 @@ sub output_enum_rst(%) { } print "\n"; } + $lineprefix = $oldprefix; output_section_rst(@_); } @@ -1899,23 +1906,29 @@ sub output_enum_rst(%) { sub output_typedef_rst(%) { my %args = %{$_[0]}; my ($parameter); - my $count; + my $oldprefix = $lineprefix; my $name = "typedef " . $args{'typedef'}; ### FIXME: should the name below contain "typedef" or not? print "\n\n.. c:type:: " . $name . "\n\n"; - print " " . $args{'purpose'} . "\n\n"; + $lineprefix = " "; + output_highlight_rst($args{'purpose'}); + print "\n"; + $lineprefix = $oldprefix; output_section_rst(@_); } sub output_struct_rst(%) { my %args = %{$_[0]}; my ($parameter); + my $oldprefix = $lineprefix; my $name = $args{'type'} . " " . $args{'struct'}; print "\n\n.. c:type:: " . $name . "\n\n"; - print " " . $args{'purpose'} . "\n\n"; + $lineprefix = " "; + output_highlight_rst($args{'purpose'}); + print "\n"; print ":Definition:\n\n"; print " ::\n\n"; @@ -1944,6 +1957,7 @@ sub output_struct_rst(%) { print " };\n\n"; print ":Members:\n\n"; + $lineprefix = " "; foreach $parameter (@{$args{'parameterlist'}}) { ($parameter =~ /^#/) && next; @@ -1953,13 +1967,12 @@ sub output_struct_rst(%) { ($args{'parameterdescs'}{$parameter_name} ne $undescribed) || next; $type = $args{'parametertypes'}{$parameter}; print " `$type $parameter`" . "\n"; - my $oldprefix = $lineprefix; - $lineprefix = " "; output_highlight_rst($args{'parameterdescs'}{$parameter_name}); - $lineprefix = $oldprefix; print "\n"; } print "\n"; + + $lineprefix = $oldprefix; output_section_rst(@_); } From 13901ef27c354e1bab49a30184ae3b96d96e521a Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 08:57:29 +0300 Subject: [PATCH 20/38] kernel-doc: do not regard $, %, or & prefixes as special in section names The use of these is confusing in the script, and per this grep, they're not used anywhere anyway: $ git grep " \* [%$&][a-zA-Z0-9_]*:" -- *.[ch] | grep -v "\$\(Id\|Revision\|Date\)" While at it, throw out the constants array, nothing is ever put there again. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 76bad55c031ec..f795660dfc7be 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -396,14 +396,12 @@ my $inline_doc_state; # 'function', 'struct', 'union', 'enum', 'typedef' my $decl_type; -my $doc_special = "\@\%\$\&"; - my $doc_start = '^/\*\*\s*$'; # Allow whitespace at end of comment start. my $doc_end = '\*/'; my $doc_com = '\s*\*\s*'; my $doc_com_body = '\s*\* ?'; my $doc_decl = $doc_com . '(\w+)'; -my $doc_sect = $doc_com . '([' . $doc_special . ']?[\w\s]+):(.*)'; +my $doc_sect = $doc_com . '(\@?[\w\s]+):(.*)'; my $doc_content = $doc_com_body . '(.*)'; my $doc_block = $doc_com . 'DOC:\s*(.*)?'; my $doc_inline_start = '^\s*/\*\*\s*$'; @@ -411,7 +409,6 @@ my $doc_inline_sect = '\s*\*\s*(@[\w\s]+):(.*)'; my $doc_inline_end = '^\s*\*/\s*$'; my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;'; -my %constants; my %parameterdescs; my @parameterlist; my %sections; @@ -511,11 +508,7 @@ sub dump_section { my $name = shift; my $contents = join "\n", @_; - if ($name =~ m/$type_constant/) { - $name = $1; -# print STDERR "constant section '$1' = '$contents'\n"; - $constants{$name} = $contents; - } elsif ($name =~ m/$type_param/) { + if ($name =~ m/$type_param/) { # print STDERR "parameter def '$1' = '$contents'\n"; $name = $1; $parameterdescs{$name} = $contents; @@ -2528,7 +2521,6 @@ sub dump_function($$) { sub reset_state { $function = ""; - %constants = (); %parameterdescs = (); %parametertypes = (); @parameterlist = (); @@ -2924,7 +2916,6 @@ sub process_file($) { dump_doc_section($file, $section, xml_escape($contents)); $contents = ""; $function = ""; - %constants = (); %parameterdescs = (); %parametertypes = (); @parameterlist = (); @@ -2942,7 +2933,6 @@ sub process_file($) { dump_doc_section($file, $section, xml_escape($contents)); $contents = ""; $function = ""; - %constants = (); %parameterdescs = (); %parametertypes = (); @parameterlist = (); From a0b96c2dbdb2c2511af407a2657d580f16c3b6f1 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 22:04:06 +0300 Subject: [PATCH 21/38] kernel-doc: fix wrong code indentation No functional changes. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index f795660dfc7be..c154c3205df17 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -2878,7 +2878,7 @@ sub process_file($) { substr($contents, 0, 1) eq "\t") { $contents = substr($contents, 1); } - $contents .= "\n"; + $contents .= "\n"; } $inline_doc_state = STATE_INLINE_TEXT; # Documentation block end */ From 830066a7a317e3e8872cb2d21dd24af0815f51f9 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 22:04:33 +0300 Subject: [PATCH 22/38] kernel-doc/rst: blank lines in output are not needed Current approach leads to two blank lines, while one is enough. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index c154c3205df17..a89ff3ca366ca 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1786,12 +1786,7 @@ sub output_highlight_rst { die $@ if $@; foreach $line (split "\n", $contents) { - if ($line eq "") { - print $lineprefix, $blankline; - } else { - print $lineprefix, $line; - } - print "\n"; + print $lineprefix . $line . "\n"; } } From 6450c8957ee3a8f58191c2ed6c5b71c7b7d1b310 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 22:04:42 +0300 Subject: [PATCH 23/38] kernel-doc: strip leading blank lines from inline doc comments The inline member markup allows whitespace lines before the actual documentation starts. Strip the leading blank lines. This improves the rst output. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index a89ff3ca366ca..e8ea295567a3a 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -2889,6 +2889,10 @@ sub process_file($) { } elsif (/$doc_content/) { if ($inline_doc_state == STATE_INLINE_TEXT) { $contents .= $1 . "\n"; + # nuke leading blank lines + if ($contents =~ /^\s*$/) { + $contents = ""; + } } elsif ($inline_doc_state == STATE_INLINE_NAME) { $inline_doc_state = STATE_INLINE_ERROR; print STDERR "Warning(${file}:$.): "; From ecbcfba126e857de8dd4996fe31fad782dd6bae0 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Thu, 26 May 2016 18:30:27 +0300 Subject: [PATCH 24/38] kernel-doc/rst: change the output layout Move away from field lists, and simply use **strong emphasis** for section headings on lines of their own. Do not use rst section headings, because their nesting depth depends on the surrounding context, which kernel-doc has no knowledge of. Also, they do not need to end up in any table of contexts or indexes. There are two related immediate benefits. Field lists are typically rendered in two columns, while the new style uses the horizontal width better. With no extra indent on the left, there's no need to be as fussy about it. Field lists are more susceptible to indentation problems than the new style. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index e8ea295567a3a..4f559de8b173a 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1823,23 +1823,23 @@ sub output_function_rst(%) { output_highlight_rst($args{'purpose'}); print "\n"; - print ":Parameters:\n\n"; - $lineprefix = " "; + print "**Parameters**\n\n"; + $lineprefix = " "; foreach $parameter (@{$args{'parameterlist'}}) { my $parameter_name = $parameter; #$parameter_name =~ s/\[.*//; $type = $args{'parametertypes'}{$parameter}; if ($type ne "") { - print " ``$type $parameter``\n"; + print "``$type $parameter``\n"; } else { - print " ``$parameter``\n"; + print "``$parameter``\n"; } if (defined($args{'parameterdescs'}{$parameter_name}) && $args{'parameterdescs'}{$parameter_name} ne $undescribed) { output_highlight_rst($args{'parameterdescs'}{$parameter_name}); } else { - print "\n _undescribed_\n"; + print " _undescribed_\n"; } print "\n"; } @@ -1852,10 +1852,10 @@ sub output_section_rst(%) { my %args = %{$_[0]}; my $section; my $oldprefix = $lineprefix; - $lineprefix = " "; + $lineprefix = ""; foreach $section (@{$args{'sectionlist'}}) { - print ":$section:\n\n"; + print "**$section**\n\n"; output_highlight_rst($args{'sections'}{$section}); print "\n"; } @@ -1875,14 +1875,14 @@ sub output_enum_rst(%) { output_highlight_rst($args{'purpose'}); print "\n"; - print "..\n\n:Constants:\n\n"; - $lineprefix = " "; + print "**Constants**\n\n"; + $lineprefix = " "; foreach $parameter (@{$args{'parameterlist'}}) { - print " `$parameter`\n"; + print "``$parameter``\n"; if ($args{'parameterdescs'}{$parameter} ne $undescribed) { output_highlight_rst($args{'parameterdescs'}{$parameter}); } else { - print " undescribed\n"; + print " _undescribed_\n"; } print "\n"; } @@ -1918,12 +1918,12 @@ sub output_struct_rst(%) { output_highlight_rst($args{'purpose'}); print "\n"; - print ":Definition:\n\n"; - print " ::\n\n"; + print "**Definition**\n\n"; + print "::\n\n"; print " " . $args{'type'} . " " . $args{'struct'} . " {\n"; foreach $parameter (@{$args{'parameterlist'}}) { if ($parameter =~ /^#/) { - print " " . "$parameter\n"; + print " " . "$parameter\n"; next; } @@ -1944,8 +1944,8 @@ sub output_struct_rst(%) { } print " };\n\n"; - print ":Members:\n\n"; - $lineprefix = " "; + print "**Members**\n\n"; + $lineprefix = " "; foreach $parameter (@{$args{'parameterlist'}}) { ($parameter =~ /^#/) && next; @@ -1954,7 +1954,7 @@ sub output_struct_rst(%) { ($args{'parameterdescs'}{$parameter_name} ne $undescribed) || next; $type = $args{'parametertypes'}{$parameter}; - print " `$type $parameter`" . "\n"; + print "``$type $parameter``\n"; output_highlight_rst($args{'parameterdescs'}{$parameter_name}); print "\n"; } From 0a7263014b3a0e05b52ccef314e2ccf6f4550a8b Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sat, 28 May 2016 14:50:20 +0300 Subject: [PATCH 25/38] kernel-doc: improve handling of whitespace on the first line param description Handle whitespace on the first line of param text as if it was the empty string. There is no need to add the newline in this case. This improves the rst output in particular, where blank lines may be problematic in parameter lists. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 4f559de8b173a..e93e796b17cef 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -2814,11 +2814,11 @@ sub process_file($) { $in_doc_sect = 1; $in_purpose = 0; $contents = $newcontents; + while ((substr($contents, 0, 1) eq " ") || + substr($contents, 0, 1) eq "\t") { + $contents = substr($contents, 1); + } if ($contents ne "") { - while ((substr($contents, 0, 1) eq " ") || - substr($contents, 0, 1) eq "\t") { - $contents = substr($contents, 1); - } $contents .= "\n"; } $section = $newsection; From b7886de43c9f8a19685cd6e81135de63b9529911 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sat, 28 May 2016 00:41:50 +0300 Subject: [PATCH 26/38] kernel-doc: strip leading whitespace from continued param descs If a param description spans multiple lines, check any leading whitespace in the first continuation line, and remove same amount of whitespace from following lines. This allows indentation in the multi-line parameter descriptions for aesthetical reasons while not causing accidentally significant indentation in the rst output. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index e93e796b17cef..f6f37e71dc089 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -2701,6 +2701,7 @@ sub process_file($) { my $in_purpose = 0; my $initial_section_counter = $section_counter; my ($orig_file) = @_; + my $leading_space; if (defined($ENV{'SRCTREE'})) { $file = "$ENV{'SRCTREE'}" . "/" . $orig_file; @@ -2822,6 +2823,7 @@ sub process_file($) { $contents .= "\n"; } $section = $newsection; + $leading_space = undef; } elsif (/$doc_end/) { if (($contents ne "") && ($contents ne "\n")) { dump_section($file, $section, xml_escape($contents)); @@ -2856,7 +2858,19 @@ sub process_file($) { $declaration_purpose .= " " . xml_escape($1); $declaration_purpose =~ s/\s+/ /g; } else { - $contents .= $1 . "\n"; + my $cont = $1; + if ($section =~ m/^@/ || $section eq $section_context) { + if (!defined $leading_space) { + if ($cont =~ m/^(\s+)/) { + $leading_space = $1; + } else { + $leading_space = ""; + } + } + + $cont =~ s/^$leading_space//; + } + $contents .= $cont . "\n"; } } else { # i dont know - bad line? ignore. From d4b08e0cd2d74eb652d580607982d5054dc42991 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sat, 28 May 2016 00:48:17 +0300 Subject: [PATCH 27/38] kernel-doc/rst: use *undescribed* instead of _undescribed_ The latter isn't special to rst. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index f6f37e71dc089..19cee0cd53a36 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1839,7 +1839,7 @@ sub output_function_rst(%) { $args{'parameterdescs'}{$parameter_name} ne $undescribed) { output_highlight_rst($args{'parameterdescs'}{$parameter_name}); } else { - print " _undescribed_\n"; + print " *undescribed*\n"; } print "\n"; } @@ -1882,7 +1882,7 @@ sub output_enum_rst(%) { if ($args{'parameterdescs'}{$parameter} ne $undescribed) { output_highlight_rst($args{'parameterdescs'}{$parameter}); } else { - print " _undescribed_\n"; + print " *undescribed*\n"; } print "\n"; } From cddfe325afedb67a15fbe1a91e82ffed40236413 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sat, 28 May 2016 10:48:37 +0300 Subject: [PATCH 28/38] kernel-doc/rst: remove fixme comment Yes, for our purposes the type should contain typedef. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 19cee0cd53a36..425a94be04f6c 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -1897,7 +1897,6 @@ sub output_typedef_rst(%) { my $oldprefix = $lineprefix; my $name = "typedef " . $args{'typedef'}; - ### FIXME: should the name below contain "typedef" or not? print "\n\n.. c:type:: " . $name . "\n\n"; $lineprefix = " "; output_highlight_rst($args{'purpose'}); From f624adef3d0b9975076c1ba7549b81ed19e34437 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sun, 29 May 2016 11:35:28 +0300 Subject: [PATCH 29/38] kernel-doc: limit the "section header:" detection to a select few kernel-doc currently identifies anything matching "section header:" (specifically a string of word characters and spaces followed by a colon) as a new section in the documentation comment, and renders the section header accordingly. Unfortunately, this turns all uses of colon into sections, mostly unintentionally. Considering the output, erroneously creating sections when not intended is always worse than erroneously not creating sections when intended. For example, a line with "http://example.com" turns into a "http" heading followed by "//example.com" in normal text style, which is quite ugly. OTOH, "WARNING: Beware of the Leopard" is just fine even if "WARNING" does not turn into a heading. It is virtually impossible to change all the kernel-doc comments, either way. The compromise is to pick the most commonly used and depended on section headers (with variants) and accept them as section headers. The accepted section headers are, case insensitive: * description: * context: * return: * returns: Additionally, case sensitive: * @return: All of the above are commonly used in the kernel-doc comments, and will result in worse output if not identified as section headers. Also, kernel-doc already has some special handling for all of them, so there's nothing particularly controversial in adding more special treatment for them. While at it, improve the whitespace handling surrounding section names. Do not consider the whitespace as part of the name. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 425a94be04f6c..20136564f2649 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -401,7 +401,8 @@ my $doc_end = '\*/'; my $doc_com = '\s*\*\s*'; my $doc_com_body = '\s*\* ?'; my $doc_decl = $doc_com . '(\w+)'; -my $doc_sect = $doc_com . '(\@?[\w\s]+):(.*)'; +# @params and a strictly limited set of supported section names +my $doc_sect = $doc_com . '\s*(\@\w+|description|context|returns?)\s*:(.*)'; my $doc_content = $doc_com_body . '(.*)'; my $doc_block = $doc_com . 'DOC:\s*(.*)?'; my $doc_inline_start = '^\s*/\*\*\s*$'; @@ -417,6 +418,8 @@ my $sectcheck; my $struct_actual; my $contents = ""; + +# the canonical section names. see also $doc_sect above. my $section_default = "Description"; # default section my $section_intro = "Introduction"; my $section = $section_default; @@ -2798,10 +2801,22 @@ sub process_file($) { $state = STATE_NORMAL; } } elsif ($state == STATE_FIELD) { # look for head: lines, and include content - if (/$doc_sect/o) { + if (/$doc_sect/i) { # case insensitive for supported section names $newsection = $1; $newcontents = $2; + # map the supported section names to the canonical names + if ($newsection =~ m/^description$/i) { + $newsection = $section_default; + } elsif ($newsection =~ m/^context$/i) { + $newsection = $section_context; + } elsif ($newsection =~ m/^returns?$/i) { + $newsection = $section_return; + } elsif ($newsection =~ m/^\@return$/) { + # special: @return is a section, not a param description + $newsection = $section_return; + } + if (($contents ne "") && ($contents ne "\n")) { if (!$in_doc_sect && $verbose) { print STDERR "${file}:$.: warning: contents before sections\n"; From 32217761ee9db0215350dfe1ca4e66f312fb8c54 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Sun, 29 May 2016 09:40:44 +0300 Subject: [PATCH 30/38] kernel-doc: concatenate contents of colliding sections If there are multiple sections with the same section name, the current implementation results in several sections by the same heading, with the content duplicated from the last section to all. Even if there's the error message, a more graceful approach is to combine all the identically named sections into one, with concatenated contents. With the supported sections already limited to select few, there are massively fewer collisions than there used to be, but this is still useful for e.g. when function parameters are documented in the middle of a documentation comment, with description spread out above and below. (This is not a recommended documentation style, but used in the kernel nonetheless.) We can now also demote the error to a warning. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 20136564f2649..3ac4b57ed76a5 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -524,11 +524,13 @@ sub dump_section { } else { # print STDERR "other section '$name' = '$contents'\n"; if (defined($sections{$name}) && ($sections{$name} ne "")) { - print STDERR "${file}:$.: error: duplicate section name '$name'\n"; - ++$errors; + print STDERR "${file}:$.: warning: duplicate section name '$name'\n"; + ++$warnings; + $sections{$name} .= $contents; + } else { + $sections{$name} = $contents; + push @sectionlist, $name; } - $sections{$name} = $contents; - push @sectionlist, $name; } } From 2f4ad40a05265827848200689094348363027069 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Mon, 30 May 2016 11:21:06 +0300 Subject: [PATCH 31/38] kernel-doc: reset contents and section harder If the documentation comment does not have params or sections, the section heading may leak from the previous documentation comment. Signed-off-by: Jani Nikula --- scripts/kernel-doc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 3ac4b57ed76a5..0eb2e7b5bf101 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -2763,6 +2763,8 @@ sub process_file($) { } $state = STATE_FIELD; + $contents = ""; + $section = $section_default; if (/-(.*)/) { # strip leading/trailing/multiple spaces $descr= $1; @@ -2960,6 +2962,7 @@ sub process_file($) { elsif (/$doc_end/) { dump_doc_section($file, $section, xml_escape($contents)); + $section = $section_default; $contents = ""; $function = ""; %parameterdescs = (); From ba35018593547eac69998e2571fc95cf620f2220 Mon Sep 17 00:00:00 2001 From: Jani Nikula Date: Tue, 31 May 2016 18:09:12 +0300 Subject: [PATCH 32/38] Documentation/sphinx: fix kernel-doc extension on python3 Reconcile differences between python2 and python3 on dealing with stdout, stderr from Popen. This fixes "name 'unicode' is not defined" errors on python3. We'll need to try to keep the extension working on both python-sphinx and python3-sphinx so we don't need two copies. Reported-and-tested-by: Marius Vlad Signed-off-by: Jani Nikula --- Documentation/sphinx/kernel-doc.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Documentation/sphinx/kernel-doc.py b/Documentation/sphinx/kernel-doc.py index 87a1332fe934f..a6e170872c8aa 100644 --- a/Documentation/sphinx/kernel-doc.py +++ b/Documentation/sphinx/kernel-doc.py @@ -23,6 +23,9 @@ # # Authors: # Jani Nikula +# +# Please make sure this works on both python2 and python3. +# import os import subprocess @@ -75,8 +78,10 @@ def run(self): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) out, err = p.communicate() - # assume the kernel sources are utf-8 - out, err = unicode(out, 'utf-8'), unicode(err, 'utf-8') + # python2 needs conversion to unicode. + # python3 with universal_newlines=True returns strings. + if sys.version_info.major < 3: + out, err = unicode(out, 'utf-8'), unicode(err, 'utf-8') if p.returncode != 0: sys.stderr.write(err) From 9cc1a44c03113cb2f3be73650d7c611c3e273ada Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Wed, 1 Jun 2016 11:12:42 +0200 Subject: [PATCH 33/38] doc/sphinx: Pass right filename as source With this error output becomes almost readable. The line numbers are still totally bonghits, but that's a lot harder to pull out of kerneldoc. We'd essentially have to insert some special markers in the kernel-doc output, split the output along these markers and then insert each block separately using state_machine.insert_input(block, source, first_line) Cc: Jani Nikula Cc: linux-doc@vger.kernel.org Cc: Jonathan Corbet Signed-off-by: Daniel Vetter Signed-off-by: Jani Nikula --- Documentation/sphinx/kernel-doc.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Documentation/sphinx/kernel-doc.py b/Documentation/sphinx/kernel-doc.py index a6e170872c8aa..9fc2c27916a9c 100644 --- a/Documentation/sphinx/kernel-doc.py +++ b/Documentation/sphinx/kernel-doc.py @@ -57,7 +57,7 @@ def run(self): env.note_dependency(os.path.abspath(filename)) tab_width = self.options.get('tab-width', self.state.document.settings.tab_width) - source = self.state_machine.input_lines.source(self.lineno - self.state_machine.input_offset - 1) + source = filename # FIXME: make this nicer and more robust against errors if 'export' in self.options: From ebff7f929b2a72fa614f5e95fd34c56c82ac9c36 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Wed, 1 Jun 2016 23:46:23 +0200 Subject: [PATCH 34/38] scripts/kernel-doc: Remove duplicated DOC: start handling Further up in the state machinery we switch from STATE_NAME to STATE_DOCBLOCK when we match /$doc_block/. Which means this block of code here is entirely unreachable, unless there are multiple DOC: sections within a single kernel-doc comment. Getting a list of all the files with more than one DOC: section using $ git grep -c " * DOC:" | grep -v ":1$" and then doing a full audit of them reveals there are no such comment blocks in the kernel. Supporting multiple DOC: sections in a single kernel-doc comment does not seem like a recommended way of doing things anyway, so nuke the code for simplicity. Cc: Jani Nikula Cc: linux-doc@vger.kernel.org Cc: Jonathan Corbet Signed-off-by: Daniel Vetter [Jani: amended the commit message] Signed-off-by: Jani Nikula --- scripts/kernel-doc | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 0eb2e7b5bf101..9fb26d142a56f 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -2942,24 +2942,7 @@ sub process_file($) { process_state3_type($_, $file); } } elsif ($state == STATE_DOCBLOCK) { - # Documentation block - if (/$doc_block/) { - dump_doc_section($file, $section, xml_escape($contents)); - $contents = ""; - $function = ""; - %parameterdescs = (); - %parametertypes = (); - @parameterlist = (); - %sections = (); - @sectionlist = (); - $prototype = ""; - if ( $1 eq "" ) { - $section = $section_intro; - } else { - $section = $1; - } - } - elsif (/$doc_end/) + if (/$doc_end/) { dump_doc_section($file, $section, xml_escape($contents)); $section = $section_default; From 16e161c8c67ac3b27cfc096cf07af9bceb144707 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Thu, 2 Jun 2016 14:59:18 +0200 Subject: [PATCH 35/38] doc/sphinx: Stop touching state_machine internals Instead of just forcefully inserting our kernel-doc input and letting the state machine stumble over it the recommended way is to create ViewList, parse that and then return the list of parsed nodes. Suggested by Jani. Cc: Jani Nikula Cc: linux-doc@vger.kernel.org Cc: Jonathan Corbet Signed-off-by: Daniel Vetter Signed-off-by: Jani Nikula --- Documentation/sphinx/kernel-doc.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/Documentation/sphinx/kernel-doc.py b/Documentation/sphinx/kernel-doc.py index 9fc2c27916a9c..bd422870101e8 100644 --- a/Documentation/sphinx/kernel-doc.py +++ b/Documentation/sphinx/kernel-doc.py @@ -32,6 +32,7 @@ import sys from docutils import nodes, statemachine +from docutils.statemachine import ViewList from docutils.parsers.rst import directives from sphinx.util.compat import Directive @@ -92,8 +93,14 @@ def run(self): sys.stderr.write(err) lines = statemachine.string2lines(out, tab_width, convert_whitespace=True) - self.state_machine.insert_input(lines, source) - return [] + result = ViewList(lines, source) + + node = nodes.section() + node.document = self.state.document + self.state.nested_parse(result, self.content_offset, node) + + return node.children + except Exception as e: env.app.warn('kernel-doc \'%s\' processing failed with: %s' % (" ".join(cmd), str(e))) From b7afa92b55043b71a37a2f658553d3e260859859 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Wed, 1 Jun 2016 23:46:24 +0200 Subject: [PATCH 36/38] scripts/kernel-doc: Also give functions symbolic names state3 = prototype parsing, so name them accordingly. Cc: Jani Nikula Cc: linux-doc@vger.kernel.org Cc: Jonathan Corbet Signed-off-by: Daniel Vetter Signed-off-by: Jani Nikula --- scripts/kernel-doc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 9fb26d142a56f..4da6f952d18ba 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -2593,7 +2593,7 @@ sub syscall_munge() { } } -sub process_state3_function($$) { +sub process_proto_function($$) { my $x = shift; my $file = shift; @@ -2623,7 +2623,7 @@ sub process_state3_function($$) { } } -sub process_state3_type($$) { +sub process_proto_type($$) { my $x = shift; my $file = shift; @@ -2937,9 +2937,9 @@ sub process_file($) { $state = STATE_INLINE; $inline_doc_state = STATE_INLINE_NAME; } elsif ($decl_type eq 'function') { - process_state3_function($_, $file); + process_proto_function($_, $file); } else { - process_state3_type($_, $file); + process_proto_type($_, $file); } } elsif ($state == STATE_DOCBLOCK) { if (/$doc_end/) From 0b0f5f29b282b18d26ce698e1aab0267234f77bf Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Fri, 3 Jun 2016 22:21:34 +0200 Subject: [PATCH 37/38] scripts/kernel-doc: Add option to inject line numbers Opt-in since this wreaks the rst output and must be removed by consumers again. This is useful to adjust the linenumbers for included kernel-doc snippets in shinx. With that sphinx error message will be accurate when there's issues with the rst-ness of the kernel-doc comments. Especially when transitioning a new docbook .tmpl to .rst this is extremely useful, since you can just use your editors compilation quickfix list to accurately jump from error to error. v2: - Also make sure that we filter the LINENO for purpose/at declaration start so it only shows for selected blocks, not all of them (Jani). While at it make it a notch more accurate. - Avoid undefined $lineno issues. I tried filtering these out at the callsite, but Jani spotted more when linting the entire kernel. Unamed unions and similar things aren't stored consistently and end up with an undefined line number (but also no kernel-doc text, just the parameter type). Simplify things and filter undefined line numbers in print_lineno() to catch them all. v3: Fix LINENO 0 issue for kernel-doc comments without @param: lines or any other special sections that directly jump to the description after the "name - purpose" line. Only really possible for functions without parameters. Noticed by Jani. Cc: Jani Nikula Cc: linux-doc@vger.kernel.org Cc: Jonathan Corbet Signed-off-by: Daniel Vetter Signed-off-by: Jani Nikula --- scripts/kernel-doc | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/scripts/kernel-doc b/scripts/kernel-doc index 4da6f952d18ba..5192213c5005a 100755 --- a/scripts/kernel-doc +++ b/scripts/kernel-doc @@ -74,6 +74,8 @@ Output selection (mutually exclusive): Output selection modifiers: -no-doc-sections Do not output DOC: sections. + -enable-lineno Enable output of #define LINENO lines. Only works with + reStructuredText format. Other parameters: -v Verbose output, more warnings and other information. @@ -319,6 +321,7 @@ my $verbose = 0; my $output_mode = "man"; my $output_preformatted = 0; my $no_doc_sections = 0; +my $enable_lineno = 0; my @highlights = @highlights_man; my $blankline = $blankline_man; my $modulename = "Kernel API"; @@ -351,6 +354,7 @@ my $man_date = ('January', 'February', 'March', 'April', 'May', 'June', # CAVEAT EMPTOR! Some of the others I localised may not want to be, which # could cause "use of undefined value" or other bugs. my ($function, %function_table, %parametertypes, $declaration_purpose); +my $declaration_start_line; my ($type, $declaration_name, $return_type); my ($newsection, $newcontents, $prototype, $brcount, %source_map); @@ -411,13 +415,16 @@ my $doc_inline_end = '^\s*\*/\s*$'; my $export_symbol = '^\s*EXPORT_SYMBOL(_GPL)?\s*\(\s*(\w+)\s*\)\s*;'; my %parameterdescs; +my %parameterdesc_start_lines; my @parameterlist; my %sections; my @sectionlist; +my %section_start_lines; my $sectcheck; my $struct_actual; my $contents = ""; +my $new_start_line = 0; # the canonical section names. see also $doc_sect above. my $section_default = "Description"; # default section @@ -486,6 +493,8 @@ while ($ARGV[0] =~ m/^-(.*)/) { usage(); } elsif ($cmd eq '-no-doc-sections') { $no_doc_sections = 1; + } elsif ($cmd eq '-enable-lineno') { + $enable_lineno = 1; } elsif ($cmd eq '-show-not-found') { $show_not_found = 1; } @@ -503,6 +512,13 @@ sub get_kernel_version() { return $version; } +# +sub print_lineno { + my $lineno = shift; + if ($enable_lineno && defined($lineno)) { + print "#define LINENO " . $lineno . "\n"; + } +} ## # dumps section contents to arrays/hashes intended for that purpose. # @@ -516,11 +532,15 @@ sub dump_section { $name = $1; $parameterdescs{$name} = $contents; $sectcheck = $sectcheck . $name . " "; + $parameterdesc_start_lines{$name} = $new_start_line; + $new_start_line = 0; } elsif ($name eq "@\.\.\.") { # print STDERR "parameter def '...' = '$contents'\n"; $name = "..."; $parameterdescs{$name} = $contents; $sectcheck = $sectcheck . $name . " "; + $parameterdesc_start_lines{$name} = $new_start_line; + $new_start_line = 0; } else { # print STDERR "other section '$name' = '$contents'\n"; if (defined($sections{$name}) && ($sections{$name} ne "")) { @@ -530,6 +550,8 @@ sub dump_section { } else { $sections{$name} = $contents; push @sectionlist, $name; + $section_start_lines{$name} = $new_start_line; + $new_start_line = 0; } } } @@ -1775,6 +1797,7 @@ sub output_blockhead_rst(%) { if ($output_selection != OUTPUT_INCLUDE) { print "**$section**\n\n"; } + print_lineno($section_start_lines{$section}); output_highlight_rst($args{'sections'}{$section}); print "\n"; } @@ -1824,6 +1847,7 @@ sub output_function_rst(%) { } } print ")\n\n"; + print_lineno($declaration_start_line); $lineprefix = " "; output_highlight_rst($args{'purpose'}); print "\n"; @@ -1840,6 +1864,9 @@ sub output_function_rst(%) { } else { print "``$parameter``\n"; } + + print_lineno($parameterdesc_start_lines{$parameter_name}); + if (defined($args{'parameterdescs'}{$parameter_name}) && $args{'parameterdescs'}{$parameter_name} ne $undescribed) { output_highlight_rst($args{'parameterdescs'}{$parameter_name}); @@ -1861,6 +1888,7 @@ sub output_section_rst(%) { foreach $section (@{$args{'sectionlist'}}) { print "**$section**\n\n"; + print_lineno($section_start_lines{$section}); output_highlight_rst($args{'sections'}{$section}); print "\n"; } @@ -1876,6 +1904,7 @@ sub output_enum_rst(%) { my $name = "enum " . $args{'enum'}; print "\n\n.. c:type:: " . $name . "\n\n"; + print_lineno($declaration_start_line); $lineprefix = " "; output_highlight_rst($args{'purpose'}); print "\n"; @@ -1903,6 +1932,7 @@ sub output_typedef_rst(%) { my $name = "typedef " . $args{'typedef'}; print "\n\n.. c:type:: " . $name . "\n\n"; + print_lineno($declaration_start_line); $lineprefix = " "; output_highlight_rst($args{'purpose'}); print "\n"; @@ -1918,6 +1948,7 @@ sub output_struct_rst(%) { my $name = $args{'type'} . " " . $args{'struct'}; print "\n\n.. c:type:: " . $name . "\n\n"; + print_lineno($declaration_start_line); $lineprefix = " "; output_highlight_rst($args{'purpose'}); print "\n"; @@ -1958,6 +1989,7 @@ sub output_struct_rst(%) { ($args{'parameterdescs'}{$parameter_name} ne $undescribed) || next; $type = $args{'parametertypes'}{$parameter}; + print_lineno($parameterdesc_start_lines{$parameter_name}); print "``$type $parameter``\n"; output_highlight_rst($args{'parameterdescs'}{$parameter_name}); print "\n"; @@ -2745,11 +2777,14 @@ sub process_file($) { if (/$doc_start/o) { $state = STATE_NAME; # next line is always the function name $in_doc_sect = 0; + $declaration_start_line = $. + 1; } } elsif ($state == STATE_NAME) {# this line is the function name (always) if (/$doc_block/o) { $state = STATE_DOCBLOCK; $contents = ""; + $new_start_line = $. + 1; + if ( $1 eq "" ) { $section = $section_intro; } else { @@ -2763,8 +2798,11 @@ sub process_file($) { } $state = STATE_FIELD; + # if there's no @param blocks need to set up default section + # here $contents = ""; $section = $section_default; + $new_start_line = $. + 1; if (/-(.*)/) { # strip leading/trailing/multiple spaces $descr= $1; @@ -2833,6 +2871,7 @@ sub process_file($) { $in_doc_sect = 1; $in_purpose = 0; $contents = $newcontents; + $new_start_line = $.; while ((substr($contents, 0, 1) eq " ") || substr($contents, 0, 1) eq "\t") { $contents = substr($contents, 1); @@ -2866,6 +2905,7 @@ sub process_file($) { dump_section($file, $section, xml_escape($contents)); $section = $section_default; $contents = ""; + $new_start_line = $.; } else { $contents .= "\n"; } @@ -2900,6 +2940,7 @@ sub process_file($) { if ($inline_doc_state == STATE_INLINE_NAME && /$doc_inline_sect/o) { $section = $1; $contents = $2; + $new_start_line = $.; if ($contents ne "") { while ((substr($contents, 0, 1) eq " ") || substr($contents, 0, 1) eq "\t") { From d90368f2fa7ded7c56d214aef087e88bba5199e7 Mon Sep 17 00:00:00 2001 From: Daniel Vetter Date: Fri, 3 Jun 2016 22:21:35 +0200 Subject: [PATCH 38/38] doc/sphinx: Track line-number of starting blocks Design is pretty simple: kernel-doc inserts breadcrumbs with line numbers, and sphinx picks them up. At first I went with a sphinx comment, but inserting those at random places seriously upsets the parser, and must be filtered. Hence why this version now uses "#define LINEO " since one of these ever escape into output it's pretty clear there is a bug. It seems to work well, and at least the 2-3 errors where sphinx complained about something that was not correct in kernel-doc text the line numbers matched up perfectly. v2: Instead of noodling around in the parser state machine, create a ViewList and parse it ourselves. This seems to be the recommended way, per Jani's suggestion. v3: - Split out ViewList pach. Splitting the kernel-doc changes from the sphinx ones isn't possible, since emitting the LINENO lines wreaks havoc with the rst formatting. We must filter them. - Improve the regex per Jani's suggestions, and compile it just once for speed. - Now that LINENO lines are eaten, also add them to function parameter descriptions. Much less content and offset than for in-line struct member descriptions, but still nice to know which exact continuation line upsets sphinx. - Simplify/clarify the line +/-1 business a bit. v4: Split out the scripts/kernel-doc changes and make line-numbers opt-in, as suggested by Jani. Cc: Jani Nikula Cc: linux-doc@vger.kernel.org Cc: Jonathan Corbet Signed-off-by: Daniel Vetter Signed-off-by: Jani Nikula --- Documentation/sphinx/kernel-doc.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/Documentation/sphinx/kernel-doc.py b/Documentation/sphinx/kernel-doc.py index bd422870101e8..4adfb0e91ecca 100644 --- a/Documentation/sphinx/kernel-doc.py +++ b/Documentation/sphinx/kernel-doc.py @@ -30,6 +30,7 @@ import os import subprocess import sys +import re from docutils import nodes, statemachine from docutils.statemachine import ViewList @@ -50,7 +51,7 @@ class KernelDocDirective(Directive): def run(self): env = self.state.document.settings.env - cmd = [env.config.kerneldoc_bin, '-rst'] + cmd = [env.config.kerneldoc_bin, '-rst', '-enable-lineno'] filename = env.config.kerneldoc_srctree + '/' + self.arguments[0] @@ -93,7 +94,19 @@ def run(self): sys.stderr.write(err) lines = statemachine.string2lines(out, tab_width, convert_whitespace=True) - result = ViewList(lines, source) + result = ViewList() + + lineoffset = 0; + line_regex = re.compile("^#define LINENO ([0-9]+)$") + for line in lines: + match = line_regex.search(line) + if match: + # sphinx counts lines from 0 + lineoffset = int(match.group(1)) - 1 + # we must eat our comments since the upset the markup + else: + result.append(line, source, lineoffset) + lineoffset += 1 node = nodes.section() node.document = self.state.document