Support FLINT's new doc parser

bin/all_rst_to_pxd.sh

@@ -4,11 +4,12 @@ set -e
 
 if [ $# -eq 0 ]
   then
-    echo "Usage: bin/all_rst_to_pxd.sh /path/to/flint/doc/source"
+    echo "Usage: bin/all_rst_to_pxd.sh /path/to/flint/doc/source [parser options]"
     exit 1
 fi
 
 FLINT_DOC_DIR=$1
+shift
 
 modules=(
     "flint"
@@ -18,7 +19,6 @@ modules=(
     "gr"
     "gr_domains"
     "gr_generic"
-    "gr_implementing"
     "gr_mat"
     "gr_mpoly"
     "gr_poly"
@@ -146,9 +146,10 @@ modules=(
     # "mpfr_vec"
 )
 
-for module in ${modules[@]}; do
+for module in "${modules[@]}"; do
     echo "Processing $module"
-    bin/rst_to_pxd.py flint/$module \
-        --flint-doc-dir=$FLINT_DOC_DIR \
-        > src/flint/flintlib/functions/$module.pxd
+    bin/rst_to_pxd.py "flint/$module" \
+        "$@" \
+        --flint-doc-dir="$FLINT_DOC_DIR" \
+        > "src/flint/flintlib/functions/$module.pxd"
 done

bin/rst_to_pxd.py

@@ -25,6 +25,10 @@
 
 will output a skeleton fmpz_poly.pxd to stdout
 
+For FLINT's Markdown documentation, add --use-flint-parser and point
+--flint-doc-dir at the checkout's doc/src directory. The parser is loaded
+from the adjacent doc/fdoc package. Without the flag, RST parsing is unchanged.
+
 
 You will need to configure the location of the flintlib submodule directory and the locations of the flint and
 arb doc source directories.
@@ -39,6 +43,8 @@
 
 #  recognize a function definition in rst
 is_func = re.compile(r"\.\.( )+(c:)?function( )*::")
+is_macro = re.compile(r"\.\.( )+(c:)?macro( )*::")
+typed_macro = re.compile(r"^.+[\s*][A-Za-z_]\w*\s*\(")
 # rename types to avoid python -- c name collisions
 rename_types = [
     (re.compile(r"\bfmpz\b"),"fmpz_struct"),
@@ -54,11 +60,19 @@
 import_dict = {}
 # gr_domains.rst lists functions that are in gr.h
 doc_to_header = {'flint/gr_domains': 'flint/gr'}
-void_parameter_list = re.compile(r"\(\s*void\s*\)$")
+void_parameter_list = re.compile(r"\(\s*void\s*\)")
 macro_declarations = {
     'flint/arf': {
         'ARF_PREC_EXACT': 'cdef const slong ARF_PREC_EXACT',
     },
+    'flint/fmpq': {
+        'fmpq_numref': 'fmpz * fmpq_numref(const fmpq_t x)',
+        'fmpq_denref': 'fmpz * fmpq_denref(const fmpq_t x)',
+    },
+    'flint/fmpq_poly': {
+        'fmpq_poly_numref': 'fmpz * fmpq_poly_numref(fmpq_poly_t poly)',
+        'fmpq_poly_denref': 'fmpz * fmpq_poly_denref(fmpq_poly_t poly)',
+    },
 }
 
 
@@ -127,6 +141,7 @@ def clean_types(function):
     ret = function.strip()
     for old, new in rename_types:
         ret = re.sub(old, new, ret)
+    # This also applies to callback signatures nested in parameter lists.
     # Cython expects zero-argument functions to use () rather than (void).
     ret = re.sub(void_parameter_list, "()", ret)
     return ret
@@ -142,8 +157,9 @@ def get_functions(file):
     for line in file:
         # Keep track of the macros
         # We want to give them types in cython...
-        if line.startswith('.. macro'):
+        if is_macro.match(line):
             macros.append(line.strip())
+            in_list = False
             continue
 
         if line.strip().endswith('\\'):
@@ -162,6 +178,29 @@ def get_functions(file):
     return ret, macros
 
 
+def get_macro_declarations(macros, h_name):
+    """Use documented types or explicit overrides; never infer macro types.
+
+    Cython can call a function-like C macro using a function declaration.
+    Untyped macros (which may also introduce variables or require lvalues)
+    cannot in general be represented this way.
+    """
+    functions, constants = [], []
+    overrides = macro_declarations.get(h_name, {})
+    for macro in macros:
+        signature = macro.split('::', 1)[1].strip()
+        name = signature.split('(', 1)[0].strip()
+        declaration = overrides.get(name)
+        if declaration is not None:
+            if declaration.startswith('cdef '):
+                constants.append(declaration)
+            else:
+                functions.append(clean_types(declaration))
+        elif typed_macro.match(signature):
+            functions.append(clean_types(signature))
+    return functions, constants
+
+
 def get_all_types(function_list):
     ret = set()
     for f in function_list:
@@ -170,6 +209,32 @@ def get_all_types(function_list):
     return ret
 
 
+def get_flint_functions(docdir, name):
+    """Read a Markdown module using the parser from its FLINT checkout.
+
+    docdir is FLINT's doc/src directory; fdoc lives alongside it.
+    """
+    docdir = Path(docdir).resolve()
+    sys.path.insert(0, str(docdir.parent))
+    try:
+        from fdoc.markup import BlockParser, split_front_matter
+    finally:
+        sys.path.pop(0)
+
+    path = docdir / name / 'index.md'
+    metadata, body, line = split_front_matter(path.read_text())
+    nodes = BlockParser(path=str(path)).parse(body, line)
+    functions, macros = [], []
+    for node in nodes:
+        if node.kind != 'div':
+            continue
+        if node['kind'] == 'function':
+            functions.extend(clean_types(p) for p in node['protos'])
+        elif node['kind'] == 'macro':
+            macros.extend('.. macro:: ' + p for p in node['protos'])
+    return functions, macros, metadata.get('header')
+
+
 def has_types(line, types):
     #ts = set(t for t in get_parameter_types(line) if t in types)
     #if ts:
@@ -208,46 +273,56 @@ def generate_pxd_file(h_name, opts):
     if name[:6] == "flint/":
         docdir = opts.flint_doc_dir
         name = name[6:]
-    with open(os.path.join(docdir, name + ".rst")) as f:
-        l, macros = get_functions(f)
-        unknown_types = gen_imports(l)
-        macro_defs = []
-        declared_macros = macro_declarations.get(h_name, {})
-        for macro in macros:
-            macro_name = macro.split("::", 1)[1].strip()
-            if macro_name in declared_macros:
-                macro_defs.append(declared_macros[macro_name])
-        print()
-        for t in unknown_types:
-            print("# unknown type " + t)
-        print()
-        for m in macros:
-            print("# " + m)
-        print()
-        print(r'cdef extern from "' + h_name + r'.h":')
-        for declaration in macro_defs:
-            print("    " + declaration)
-        for f in l:
-            if has_types(f, unknown_types):
-                print("    # " + f)
-            else:
-                print("    " + f)
+    if opts.use_flint_parser:
+        l, macros, header = get_flint_functions(docdir, name)
+        if header:
+            h_name = 'flint/' + header.removesuffix('.h')
+    else:
+        with open(os.path.join(docdir, name + ".rst")) as f:
+            l, macros = get_functions(f)
+
+    macro_functions, macro_defs = get_macro_declarations(macros, h_name)
+    l.extend(macro_functions)
+    if opts.use_flint_parser and name == 'gr_domains':
+        # Context predicates moved from gr_domains to gr in the Markdown docs.
+        # Preserve the import path used by bindings generated from older docs.
+        print('from flint.flintlib.functions.gr cimport *')
+    unknown_types = gen_imports(l)
+    print()
+    for t in unknown_types:
+        print("# unknown type " + t)
+    print()
+    for m in macros:
+        print("# " + m)
+    print()
+    print(r'cdef extern from "' + h_name + r'.h":')
+    for declaration in macro_defs:
+        print("    " + declaration)
+    for f in l:
+        if has_types(f, unknown_types):
+            print("    # " + f)
+        else:
+            print("    " + f)
 
 
 def main(*args):
     usage = """
     $ cd /path/to/python-flint
     $ bin/rst_to_pxd.py flint/fmpz --flint-doc-dir=/path/to/flint/doc/source
+    $ bin/rst_to_pxd.py flint/fmpz --use-flint-parser --flint-doc-dir=/path/to/flint/doc/src
     """
-    parser = argparse.ArgumentParser(description='Generate a pxd file from an rst file',
+    parser = argparse.ArgumentParser(description='Generate a pxd file from FLINT documentation',
                                      usage=usage)
     parser.add_argument('--flint-lib-dir', help='location of the flintlib submodule',
                         default="./src/flint/flintlib")
     parser.add_argument('--arb-doc-dir', help='location of the arb doc source directory',
                         default="/Users/davideinstein/projects/arb/doc/source")
     parser.add_argument('--flint-doc-dir', help='location of the flint doc source directory',
                         default="/Users/davideinstein/projects/flint2/doc/source")
-    parser.add_argument('name', help='name of the rst file to parse (e.g. flint/fmpz)')
+    parser.add_argument('--use-flint-parser', action='store_true',
+                        help="parse Markdown using FLINT's doc/fdoc parser; "
+                             "--flint-doc-dir must point to FLINT's doc/src")
+    parser.add_argument('name', help='documentation module to parse (e.g. flint/fmpz)')
 
     args = parser.parse_args()
 

src/flint/flintlib/functions/flint.pxd

@@ -45,7 +45,7 @@ cdef extern from "flint/flint.h":
     # int flint_sscanf(const char * s, const char * str, ...)
     void flint_abort()
     # void flint_throw(flint_err_t exc, const char * msg, ...)
-    # void flint_set_abort(void (* func)(void))
+    # void flint_set_abort(void (* func)())
     # void flint_set_throw(void (* func)(flint_err_t, const char *, va_list))
     # void flint_merge_sort(void * buf, slong len, slong size, int (* cmp) (const void *, const void *, void *), void * data)
     # void flint_sort(void * buf, slong len, slong size, int (* cmp) (const void *, const void *, void *), void * data)

src/flint/types/_gr.pxd

@@ -118,8 +118,6 @@ from flint.flintlib.functions.gr cimport (
     gr_is_zero,
     gr_is_one,
     gr_is_neg_one,
-    gr_is_integer,
-    gr_is_rational,
 
     gr_equal,