GNUnet  0.19.2
typescriptdomain.py
Go to the documentation of this file.
1 """
2 TypeScript domain.
3 
4 :copyright: Copyright 2019 by Taler Systems SA
5 :license: LGPLv3+
6 :author: Florian Dold
7 """
8 
9 import re
10 
11 from pathlib import Path
12 
13 from docutils import nodes
14 from typing import List, Optional, Iterable, Dict, Tuple
15 from typing import cast
16 
17 from pygments.lexers import get_lexer_by_name
18 from pygments.filter import Filter
19 from pygments.token import Literal, Text, Operator, Keyword, Name, Number
20 from pygments.token import Comment, Token, _TokenType
21 from pygments.token import *
22 from pygments.lexer import RegexLexer, bygroups, include
23 from pygments.formatters import HtmlFormatter
24 
25 from docutils import nodes
26 from docutils.nodes import Element, Node
27 
28 from sphinx.roles import XRefRole
29 from sphinx.domains import Domain, ObjType, Index
30 from sphinx.directives import directives
31 from sphinx.util.docutils import SphinxDirective
32 from sphinx.util.nodes import make_refnode
33 from sphinx.util import logging
34 from sphinx.highlighting import PygmentsBridge
35 from sphinx.builders.html import StandaloneHTMLBuilder
36 from sphinx.pygments_styles import SphinxStyle
37 
38 logger = logging.getLogger(__name__)
39 
40 
41 class TypeScriptDefinition(SphinxDirective):
42  """
43  Directive for a code block with special highlighting or line numbering
44  settings.
45  """
46 
47  has_content = True
48  required_arguments = 1
49  optional_arguments = 0
50  final_argument_whitespace = False
51  option_spec = {
52  "force": directives.flag,
53  "linenos": directives.flag,
54  "dedent": int,
55  "lineno-start": int,
56  "emphasize-lines": directives.unchanged_required,
57  "caption": directives.unchanged_required,
58  "class": directives.class_option,
59  }
60 
61  def run(self) -> List[Node]:
62  document = self.state.document
63  code = "\n".join(self.content)
64  location = self.state_machine.get_source_and_line(self.lineno)
65 
66  linespec = self.options.get("emphasize-lines")
67  if linespec:
68  try:
69  nlines = len(self.content)
70  hl_lines = parselinenos(linespec, nlines)
71  if any(i >= nlines for i in hl_lines):
72  logger.warning(
73  __("line number spec is out of range(1-%d): %r")
74  % (nlines, self.options["emphasize-lines"]),
75  location=location,
76  )
77 
78  hl_lines = [x + 1 for x in hl_lines if x < nlines]
79  except ValueError as err:
80  return [document.reporter.warning(err, line=self.lineno)]
81  else:
82  hl_lines = None
83 
84  if "dedent" in self.options:
85  location = self.state_machine.get_source_and_line(self.lineno)
86  lines = code.split("\n")
87  lines = dedent_lines(lines, self.options["dedent"], location=location)
88  code = "\n".join(lines)
89 
90  literal = nodes.literal_block(code, code) # type: Element
91  if "linenos" in self.options or "lineno-start" in self.options:
92  literal["linenos"] = True
93  literal["classes"] += self.options.get("class", [])
94  literal["force"] = "force" in self.options
95  literal["language"] = "tsref"
96  extra_args = literal["highlight_args"] = {}
97  if hl_lines is not None:
98  extra_args["hl_lines"] = hl_lines
99  if "lineno-start" in self.options:
100  extra_args["linenostart"] = self.options["lineno-start"]
101  self.set_source_info(literal)
102 
103  caption = self.options.get("caption")
104  if caption:
105  try:
106  literal = container_wrapper(self, literal, caption)
107  except ValueError as exc:
108  return [document.reporter.warning(exc, line=self.lineno)]
109 
110  tsid = "tsref-type-" + self.arguments[0]
111  literal["ids"].append(tsid)
112 
113  tsname = self.arguments[0]
114  ts = self.env.get_domain("ts")
115  ts.add_object("type", tsname, self.env.docname, tsid)
116 
117  return [literal]
118 
119 
120 class TypeScriptDomain(Domain):
121  """TypeScript domain."""
122 
123  name = "ts"
124  label = "TypeScript"
125 
126  directives = {
127  "def": TypeScriptDefinition,
128  }
129 
130  roles = {
131  "type": XRefRole(
132  lowercase=False, warn_dangling=True, innernodeclass=nodes.inline
133  ),
134  }
135 
136  dangling_warnings = {
137  "type": "undefined TypeScript type: %(target)s",
138  }
139 
140  def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode):
141  try:
142  info = self.objectsobjects[(str(typ), str(target))]
143  except KeyError:
144  logger.warn("type {}/{} not found".format(typ, target))
145  return None
146  else:
147  anchor = "tsref-type-{}".format(str(target))
148  title = typ.upper() + " " + target
149  return make_refnode(builder, fromdocname, info[0], anchor, contnode, title)
150 
151  def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode):
152  """Resolve the pending_xref *node* with the given *target*.
153 
154  The reference comes from an "any" or similar role, which means that Sphinx
155  don't know the type.
156 
157  For now sphinxcontrib-httpdomain doesn't resolve any xref nodes.
158 
159  :return:
160  list of tuples ``('domain:role', newnode)``, where ``'domain:role'``
161  is the name of a role that could have created the same reference,
162  """
163  ret = []
164  try:
165  info = self.objectsobjects[("type", str(target))]
166  except KeyError:
167  pass
168  else:
169  anchor = "tsref-type-{}".format(str(target))
170  title = "TYPE" + " " + target
171  node = make_refnode(builder, fromdocname, info[0], anchor, contnode, title)
172  ret.append(("ts:type", node))
173  return ret
174 
175  @property
176  def objects(self) -> Dict[Tuple[str, str], Tuple[str, str]]:
177  return self.data.setdefault(
178  "objects", {}
179  ) # (objtype, name) -> docname, labelid
180 
181  def add_object(self, objtype: str, name: str, docname: str, labelid: str) -> None:
182  self.objectsobjects[objtype, name] = (docname, labelid)
183 
184 
185 class BetterTypeScriptLexer(RegexLexer):
186  """
187  For `TypeScript <https://www.typescriptlang.org/>`_ source code.
188  """
189 
190  name = "TypeScript"
191  aliases = ["ts"]
192  filenames = ["*.ts"]
193  mimetypes = ["text/x-typescript"]
194 
195  flags = re.DOTALL
196  tokens = {
197  "commentsandwhitespace": [
198  (r"\s+", Text),
199  (r"<!--", Comment),
200  (r"//.*?\n", Comment.Single),
201  (r"/\*.*?\*/", Comment.Multiline),
202  ],
203  "slashstartsregex": [
204  include("commentsandwhitespace"),
205  (
206  r"/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/" r"([gim]+\b|\B)",
207  String.Regex,
208  "#pop",
209  ),
210  (r"(?=/)", Text, ("#pop", "badregex")),
211  (r"", Text, "#pop"),
212  ],
213  "badregex": [(r"\n", Text, "#pop")],
214  "typeexp": [
215  (r"[a-zA-Z0-9_?.$]+", Keyword.Type),
216  (r"\s+", Text),
217  (r"[|]", Text),
218  (r"\n", Text, "#pop"),
219  (r";", Text, "#pop"),
220  (r"", Text, "#pop"),
221  ],
222  "root": [
223  (r"^(?=\s|/|<!--)", Text, "slashstartsregex"),
224  include("commentsandwhitespace"),
225  (
226  r"\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|"
227  r"(<<|>>>?|==?|!=?|[-<>+*%&\|\^/])=?",
228  Operator,
229  "slashstartsregex",
230  ),
231  (r"[{(\[;,]", Punctuation, "slashstartsregex"),
232  (r"[})\].]", Punctuation),
233  (
234  r"(for|in|while|do|break|return|continue|switch|case|default|if|else|"
235  r"throw|try|catch|finally|new|delete|typeof|instanceof|void|"
236  r"this)\b",
237  Keyword,
238  "slashstartsregex",
239  ),
240  (
241  r"(var|let|const|with|function)\b",
242  Keyword.Declaration,
243  "slashstartsregex",
244  ),
245  (
246  r"(abstract|boolean|byte|char|class|const|debugger|double|enum|export|"
247  r"extends|final|float|goto|implements|import|int|interface|long|native|"
248  r"package|private|protected|public|short|static|super|synchronized|throws|"
249  r"transient|volatile)\b",
250  Keyword.Reserved,
251  ),
252  (r"(true|false|null|NaN|Infinity|undefined)\b", Keyword.Constant),
253  (
254  r"(Array|Boolean|Date|Error|Function|Math|netscape|"
255  r"Number|Object|Packages|RegExp|String|sun|decodeURI|"
256  r"decodeURIComponent|encodeURI|encodeURIComponent|"
257  r"Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|"
258  r"window)\b",
259  Name.Builtin,
260  ),
261  # Match stuff like: module name {...}
262  (
263  r"\b(module)(\s*)(\s*[a-zA-Z0-9_?.$][\w?.$]*)(\s*)",
264  bygroups(Keyword.Reserved, Text, Name.Other, Text),
265  "slashstartsregex",
266  ),
267  # Match variable type keywords
268  (r"\b(string|bool|number)\b", Keyword.Type),
269  # Match stuff like: constructor
270  (r"\b(constructor|declare|interface|as|AS)\b", Keyword.Reserved),
271  # Match stuff like: super(argument, list)
272  (
273  r"(super)(\s*)\‍(([a-zA-Z0-9,_?.$\s]+\s*)\‍)",
274  bygroups(Keyword.Reserved, Text),
275  "slashstartsregex",
276  ),
277  # Match stuff like: function() {...}
278  (r"([a-zA-Z_?.$][\w?.$]*)\‍(\‍) \{", Name.Other, "slashstartsregex"),
279  # Match stuff like: (function: return type)
280  (
281  r"([a-zA-Z0-9_?.$][\w?.$]*)(\s*:\s*)",
282  bygroups(Name.Other, Text),
283  "typeexp",
284  ),
285  # Match stuff like: type Foo = Bar | Baz
286  (
287  r"\b(type)(\s*)([a-zA-Z0-9_?.$]+)(\s*)(=)(\s*)",
288  bygroups(Keyword.Reserved, Text, Name.Other, Text, Operator, Text),
289  "typeexp",
290  ),
291  (r"[$a-zA-Z_][a-zA-Z0-9_]*", Name.Other),
292  (r"[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?", Number.Float),
293  (r"0x[0-9a-fA-F]+", Number.Hex),
294  (r"[0-9]+", Number.Integer),
295  (r'"(\\\\|\\"|[^"])*"', String.Double),
296  (r"'(\\\\|\\'|[^'])*'", String.Single),
297  ],
298  }
299 
300 
301 # Map from token id to props.
302 # Properties can't be added to tokens
303 # since they derive from Python's tuple.
304 token_props = {}
305 
306 
307 class LinkFilter(Filter):
308  def __init__(self, app, **options):
309  self.appapp = app
310  Filter.__init__(self, **options)
311 
312  def _filter_one_literal(self, ttype, value):
313  last = 0
314  for m in re.finditer(literal_reg, value):
315  pre = value[last : m.start()]
316  if pre:
317  yield ttype, pre
318  t = copy_token(ttype)
319  tok_setprop(t, "is_literal", True)
320  yield t, m.group(1)
321  last = m.end()
322  post = value[last:]
323  if post:
324  yield ttype, post
325 
326  def filter(self, lexer, stream):
327  for ttype, value in stream:
328  if ttype in Token.Keyword.Type:
329  t = copy_token(ttype)
330  tok_setprop(t, "xref", value.strip())
331  tok_setprop(t, "is_identifier", True)
332  yield t, value
333  elif ttype in Token.Comment:
334  last = 0
335  for m in re.finditer(link_reg, value):
336  pre = value[last : m.start()]
337  if pre:
338  yield from self._filter_one_literal_filter_one_literal(ttype, pre)
339  t = copy_token(ttype)
340  x1, x2 = m.groups()
341  x0 = m.group(0)
342  if x2 is None:
343  caption = x1.strip()
344  xref = x1.strip()
345  else:
346  caption = x1.strip()
347  xref = x2.strip()
348  tok_setprop(t, "xref", xref)
349  tok_setprop(t, "caption", caption)
350  if x0.endswith("_"):
351  tok_setprop(t, "trailing_underscore", True)
352  yield t, m.group(1)
353  last = m.end()
354  post = value[last:]
355  if post:
356  yield from self._filter_one_literal_filter_one_literal(ttype, post)
357  else:
358  yield ttype, value
359 
360 
361 _escape_html_table = {
362  ord("&"): u"&amp;",
363  ord("<"): u"&lt;",
364  ord(">"): u"&gt;",
365  ord('"'): u"&quot;",
366  ord("'"): u"&#39;",
367 }
368 
369 
370 class LinkingHtmlFormatter(HtmlFormatter):
371  def __init__(self, **kwargs):
372  super(LinkingHtmlFormatter, self).__init__(**kwargs)
373  self._builder_builder = kwargs["_builder"]
374  self._bridge_bridge = kwargs["_bridge"]
375 
376  def _get_value(self, value, tok):
377  xref = tok_getprop(tok, "xref")
378  caption = tok_getprop(tok, "caption")
379 
380  if tok_getprop(tok, "is_literal"):
381  return '<span style="font-weight: bolder">%s</span>' % (value,)
382 
383  if tok_getprop(tok, "trailing_underscore"):
384  logger.warn(
385  "{}:{}: code block contains xref to '{}' with unsupported trailing underscore".format(
386  self._bridge_bridge.path, self._bridge_bridge.line, xref
387  )
388  )
389 
390  if tok_getprop(tok, "is_identifier"):
391  if xref.startswith('"'):
392  return value
393  if re.match("^[0-9]+$", xref) is not None:
394  return value
395  if xref in (
396  "number",
397  "object",
398  "string",
399  "boolean",
400  "any",
401  "true",
402  "false",
403  "null",
404  "undefined",
405  "Array",
406  "unknown",
407  ):
408  return value
409 
410  if self._bridge_bridge.docname is None:
411  return value
412  if xref is None:
413  return value
414  content = caption if caption is not None else value
415  ts = self._builder_builder.env.get_domain("ts")
416  r1 = ts.objects.get(("type", xref), None)
417  if r1 is not None:
418  rel_uri = (
419  self._builder_builder.get_relative_uri(self._bridge_bridge.docname, r1[0])
420  + "#"
421  + r1[1]
422  )
423  return (
424  '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>'
425  % (rel_uri, content)
426  )
427 
428  std = self._builder_builder.env.get_domain("std")
429  r2 = std.labels.get(xref.lower(), None)
430  if r2 is not None:
431  rel_uri = (
432  self._builder_builder.get_relative_uri(self._bridge_bridge.docname, r2[0])
433  + "#"
434  + r2[1]
435  )
436  return (
437  '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>'
438  % (rel_uri, content)
439  )
440  r3 = std.anonlabels.get(xref.lower(), None)
441  if r3 is not None:
442  rel_uri = (
443  self._builder_builder.get_relative_uri(self._bridge_bridge.docname, r3[0])
444  + "#"
445  + r3[1]
446  )
447  return (
448  '<a style="color:inherit;text-decoration:underline" href="%s">%s</a>'
449  % (rel_uri, content)
450  )
451 
452  logger.warn(
453  "{}:{}: code block contains unresolved xref '{}'".format(
454  self._bridge_bridge.path, self._bridge_bridge.line, xref
455  )
456  )
457 
458  return value
459 
460  def _fmt(self, value, tok):
461  cls = self._get_css_class(tok)
462  value = self._get_value_get_value(value, tok)
463  if cls is None or cls == "":
464  return value
465  return '<span class="%s">%s</span>' % (cls, value)
466 
467  def _format_lines(self, tokensource):
468  """
469  Just format the tokens, without any wrapping tags.
470  Yield individual lines.
471  """
472  lsep = self.lineseparator
473  escape_table = _escape_html_table
474 
475  line = ""
476  for ttype, value in tokensource:
477  link = get_annotation(ttype, "link")
478 
479  parts = value.translate(escape_table).split("\n")
480 
481  if len(parts) == 0:
482  # empty token, usually should not happen
483  pass
484  elif len(parts) == 1:
485  # no newline before or after token
486  line += self._fmt_fmt(parts[0], ttype)
487  else:
488  line += self._fmt_fmt(parts[0], ttype)
489  yield 1, line + lsep
490  for part in parts[1:-1]:
491  yield 1, self._fmt_fmt(part, ttype) + lsep
492  line = self._fmt_fmt(parts[-1], ttype)
493 
494  if line:
495  yield 1, line + lsep
496 
497 
498 class MyPygmentsBridge(PygmentsBridge):
499  def __init__(self, builder, trim_doctest_flags):
500  self.destdest = "html"
501  self.trim_doctest_flagstrim_doctest_flags = trim_doctest_flags
502  self.formatter_argsformatter_args = {
503  "style": SphinxStyle,
504  "_builder": builder,
505  "_bridge": self,
506  }
507  self.formatterformatter = LinkingHtmlFormatter
508  self.builderbuilder = builder
509  self.pathpath = None
510  self.lineline = None
511  self.docnamedocname = None
512 
514  self, source, lang, opts=None, force=False, location=None, **kwargs
515  ):
516  if isinstance(location, tuple):
517  docname, line = location
518  self.lineline = line
519  self.pathpath = self.builderbuilder.env.doc2path(docname)
520  self.docnamedocname = docname
521  elif isinstance(location, Element):
522  self.lineline = location.line
523  self.pathpath = location.source
524  self.docnamedocname = self.builderbuilder.env.path2doc(self.pathpath)
525  return super().highlight_block(source, lang, opts, force, location, **kwargs)
526 
527 
528 class MyHtmlBuilder(StandaloneHTMLBuilder):
529  name = "html-linked"
530 
531  def init_highlighter(self):
532  if self.config.pygments_style is not None:
533  style = self.config.pygments_style
534  elif self.theme:
535  style = self.theme.get_confstr("theme", "pygments_style", "none")
536  else:
537  style = "sphinx"
538  self.highlighterhighlighter = MyPygmentsBridge(self, self.config.trim_doctest_flags)
539  self.dark_highlighterdark_highlighter = None
540 
541 
542 def get_annotation(tok, key):
543  if not hasattr(tok, "kv"):
544  return None
545  return tok.kv.get(key)
546 
547 
548 def copy_token(tok):
549  new_tok = _TokenType(tok)
550  # This part is very fragile against API changes ...
551  new_tok.subtypes = set(tok.subtypes)
552  new_tok.parent = tok.parent
553  return new_tok
554 
555 
556 def tok_setprop(tok, key, value):
557  tokid = id(tok)
558  e = token_props.get(tokid)
559  if e is None:
560  e = token_props[tokid] = (tok, {})
561  _, kv = e
562  kv[key] = value
563 
564 
565 def tok_getprop(tok, key):
566  tokid = id(tok)
567  e = token_props.get(tokid)
568  if e is None:
569  return None
570  _, kv = e
571  return kv.get(key)
572 
573 
574 link_reg = re.compile(r"(?<!`)`([^`<]+)\s*(?:<([^>]+)>)?\s*`_?")
575 literal_reg = re.compile(r"``([^`]+)``")
576 
577 
578 def setup(app):
579 
580  class TsrefLexer(BetterTypeScriptLexer):
581  def __init__(self, **options):
582  super().__init__(**options)
583  self.add_filter(LinkFilter(app))
584 
585  app.add_lexer("tsref", TsrefLexer)
586  app.add_domain(TypeScriptDomain)
587  app.add_builder(MyHtmlBuilder)
def __init__(self, app, **options)
def _filter_one_literal(self, ttype, value)
def filter(self, lexer, stream)
def __init__(self, builder, trim_doctest_flags)
def highlight_block(self, source, lang, opts=None, force=False, location=None, **kwargs)
def resolve_xref(self, env, fromdocname, builder, typ, target, node, contnode)
def resolve_any_xref(self, env, fromdocname, builder, target, node, contnode)
Dict[Tuple[str, str], Tuple[str, str]] objects(self)
None add_object(self, str objtype, str name, str docname, str labelid)
static struct GNUNET_IDENTITY_Handle * id
Handle to identity service.
static int get
Get DID Documement for DID Flag.
Definition: gnunet-did.c:66
uint16_t len
length of data (which is always a uint32_t, but presumably this can be used to specify that fewer byt...
def get_annotation(tok, key)
def tok_getprop(tok, key)
def tok_setprop(tok, key, value)