import torch
import sys
import ast
import inspect
import string
from textwrap import dedent
from typing import List
from torch._C._jit_tree_views import (
ClassDef, Ident, Stmt, Decl, Def, Var,
EmptyTypeAnnotation, Param, ExprStmt, Assign,
Delete, Return, Raise, Assert, AugAssign, While,
For, If, Pass, Break, Continue, Apply, Dots, Select,
TrueLiteral, FalseLiteral, NoneLiteral, Starred,
ListLiteral, TupleLiteral, DictLiteral, Const,
StringLiteral, ListComp, Attribute, BinOp, UnaryOp,
SliceExpr, Subscript, TernaryIf, With, WithItem, Property,
DictComp,
)
from torch._utils_internal import get_source_lines_and_file
from torch._jit_internal import SourceContext, should_drop, is_static_fn
import torch.jit.annotations
# Borrowed from cPython implementation
# https://github.com/python/cpython/blob/561612d8456cfab5672c9b445521113b847bd6b3/Lib/textwrap.py#L411#
_reserved_prefix = '__jit'
_reserved_names = {'print'}
_identifier_chars = set(string.ascii_lowercase + string.ascii_uppercase + string.digits)
def is_reserved_name(name):
return name.startswith(_reserved_prefix) or name in _reserved_names
pretty_node_names = {
ast.FunctionDef: "function definitions",
ast.For: "for loops",
ast.Delete: "del statements",
ast.ClassDef: "class definitions",
ast.With: "with statements",
ast.Raise: "raise statements",
ast.Assert: "assertions",
ast.Import: "import statements",
ast.ImportFrom: "import statements",
ast.Global: "global variables",
ast.Break: "break statements",
ast.Continue: "continue statements",
}
node_start_tokens = {
ast.FunctionDef: "def",
ast.For: "for",
ast.Delete: "del",
ast.ClassDef: "class",
ast.With: "with",
ast.Raise: "raise",
ast.Assert: "assert",
ast.Import: "import",
ast.ImportFrom: "from",
ast.Global: "global",
ast.Break: "break",
ast.Continue: "continue",
}
pretty_node_names.update({
ast.AsyncFunctionDef: "async function definitions",
ast.AsyncFor: "async for loops",
ast.AsyncWith: "async with statements",
ast.Try: "try blocks",
ast.Nonlocal: "nonlocal variables",
})
node_start_tokens.update({
ast.AsyncFunctionDef: "async def",
ast.AsyncFor: "async for",
ast.AsyncWith: "async with",
ast.Try: "try",
ast.Nonlocal: "nonlocal",
})
if sys.version_info >= (3, 6):
pretty_node_names.update({
ast.AnnAssign: "annotated assignments",
})
# NB: no specific token for AnnAssign
class FrontendError(Exception):
def __init__(self, source_range, msg):
self.source_range = source_range
self.msg = msg
# This has to be instantiated here so the ErrorReport is accurate to the
# call stack when the FrontendError was raised
self.error_report = torch._C.ErrorReport(self.source_range)
def __str__(self):
return self.msg + self.error_report.what().lstrip()
class NotSupportedError(FrontendError):
pass
class UnsupportedNodeError(NotSupportedError):
def __init__(self, ctx, offending_node, reason=''):
# If we don't have a specific token, we default to length of 1
node_type = type(offending_node)
range_len = len(node_start_tokens.get(node_type, ' '))
source_range = ctx.make_range(offending_node.lineno,
offending_node.col_offset,
offending_node.col_offset + range_len)
feature_name = pretty_node_names.get(node_type, node_type.__name__)
msg = "{} {}aren't supported".format(feature_name, reason + ' ' if reason else '')
super(UnsupportedNodeError, self).__init__(source_range, msg)
class FrontendTypeError(FrontendError):
pass
def build_withitems(ctx, items):
items = [build_withitem(ctx, i) for i in items]
return list(items)
def build_stmts(ctx, stmts):
stmts = [build_stmt(ctx, s) for s in stmts]
return list(filter(None, stmts))
def get_class_properties(cls, self_name):
"""
Get a list of Property objects representing the properties of a class.
Args:
cls: The class to get properties of.
self_name: The name of the class that the properties should belong to.
Returns:
A list of Property objects corresponding to the properties of cls. Property
here refers to the subclass of TreeView.
"""
props = inspect.getmembers(
cls, predicate=lambda m: isinstance(m, property))
# Any property that should not compiled must be in this list on the Module.
unused_properties = getattr(cls, "__jit_unused_properties__", [])
# Create Property TreeView objects from inspected property objects.
properties = []
for prop in props:
if prop[0] not in unused_properties and not should_drop(prop[1].fget):
getter = get_jit_def(prop[1].fget, f"__{prop[0]}_getter", self_name=self_name)
setter = get_jit_def(prop[1].fset, f"__{prop[0]}_setter", self_name=self_name) if prop[1].fset else None
properties.append(Property(getter.range(), Ident(getter.range(), prop[0]), getter, setter))
return properties
def get_jit_class_def(cls, self_name):
# Get defs for each method within the current class independently
# TODO: proper overriding analysis when implementing class inheritance
methods = inspect.getmembers(
cls,
predicate=lambda m: (inspect.ismethod(m) or inspect.isfunction(m))
and not is_static_fn(cls, m.__name__)
and m.__name__ in cls.__dict__
)
def is_classmethod(fn):
return inspect.ismethod(fn) and getattr(fn, "__self__", None) == cls
methods = [get_jit_def(method[1],
method[0],
self_name=self_name,
is_classmethod=is_classmethod(method[1])) for method in methods]
properties = get_class_properties(cls, self_name)
sourcelines, file_lineno, filename = get_source_lines_and_file(cls, torch._C.ErrorReport.call_stack())
source = ''.join(sourcelines)
dedent_src = dedent(source)
py_ast = ast.parse(dedent_src)
leading_whitespace_len = len(source.split('\n', 1)[0]) - len(dedent_src.split('\n', 1)[0])
ctx = SourceContext(source, filename, file_lineno, leading_whitespace_len, False)
return build_class_def(ctx, py_ast.body[0], methods, properties, self_name)
def normalize_source_lines(sourcelines: List[str]) -> List[str]:
"""
This helper function accepts a list of source lines. It finds the
indentation level of the function definition (`def`), then it indents
all lines in the function body to a point at or greater than that
level. This allows for comments and continued string literals that
are at a lower indentation than the rest of the code.
Args:
sourcelines: function source code, separated into lines by
the '\n' character
Returns:
A list of source lines that have been correctly aligned
"""
def remove_prefix(text, prefix):
return text[text.startswith(prefix) and len(prefix):]
# Find the line and line number containing the function definition
for i, l in enumerate(sourcelines):
if l.lstrip().startswith("def"):
idx = i
break
fn_def = sourcelines[idx]
# Get a string representing the amount of leading whitespace
whitespace = fn_def.split("def")[0]
# Add this leading whitespace to all lines before and after the `def`
aligned_prefix = [whitespace + remove_prefix(s, whitespace) for s in sourcelines[:idx]]
aligned_suffix = [whitespace + remove_prefix(s, whitespace) for s in sourcelines[idx + 1:]]
# Put it together again
aligned_prefix.append(fn_def)
return aligned_prefix + aligned_suffix
def get_jit_def(fn, def_name, self_name=None, is_classmethod=False):
"""
Build a JIT AST (TreeView) from the given function.
Args:
fn: A function object to compile
def_name: The name to give to the resulting AST object. This is not
always the same as `fn.__name__`, for example:
def _forward(self):
...
forward = _forward
In this case, the `__name__` attribute of the function object is "_forward",
but we want the result AST to have the name "forward".
self_name: If this function is a method, what the type name of `self` is.
"""
sourcelines, file_lineno, filename = get_source_lines_and_file(fn, torch._C.ErrorReport.call_stack())
sourcelines = normalize_source_lines(sourcelines)
source = ''.join(sourcelines)
dedent_src = dedent(source)
py_ast = ast.parse(dedent_src)
if len(py_ast.body) != 1 or not isinstance(py_ast.body[0], ast.FunctionDef):
raise RuntimeError(f"Expected a single top-level function: {filename}:{file_lineno}")
leading_whitespace_len = len(source.split('\n', 1)[0]) - len(dedent_src.split('\n', 1)[0])
type_line = torch.jit.annotations.get_type_line(source)
ctx = SourceContext(source, filename, file_lineno, leading_whitespace_len, True)
fn_def = py_ast.body[0]
if is_classmethod:
arg_name = fn_def.args.args[0].arg
# Insert a statement that assigns the first argument to the class
assign_stmt = ast.parse(f"{arg_name} = {self_name}").body[0]
fn_def.body.insert(0, assign_stmt)
# Swap out the function signature and body if it is unused
if should_drop(fn):
unused_fn_def = ast.parse("def unused_fn(self: Any):\n\traise RuntimeError(\"Cannot call @unused methods\")")
if len(unused_fn_def.body) != 1 or not isinstance(unused_fn_def.body[0], ast.FunctionDef):
raise RuntimeError(f"Expected a single top-level function: {filename}:{file_lineno}")
unused_def = unused_fn_def.body[0]
fn_def.body = unused_def.body
# kwarg/vararg not supported by `build_def`
fn_def.args.kwarg = fn_def.args.vararg = None
for arg in fn_def.args.args + fn_def.args.kwonlyargs:
# Replace potentially unsupported type annotations by "Any"
arg.annotation = unused_def.args.args[0].annotation
return build_def(ctx, fn_def, type_line, def_name, self_name=self_name)
class Builder(object):
def __call__(self, ctx, node):
method = getattr(self, 'build_' + node.__class__.__name__, None)
if method is None:
raise UnsupportedNodeError(ctx, node)
return method(ctx, node)
def build_class_def(ctx, py_def, methods, properties, self_name):
r = ctx.make_range(py_def.lineno, py_def.col_offset,
py_def.col_offset + len("class"))
return ClassDef(Ident(r, self_name), [Stmt(method) for method in methods], properties)
def build_def(ctx, py_def, type_line, def_name, self_name=None):
body = py_def.body
r = ctx.make_range(py_def.lineno + len(py_def.decorator_list),
py_def.col_offset,
py_def.col_offset + len("def"))
param_list = build_param_list(ctx, py_def.args, self_name)
return_type = None
if getattr(py_def, 'returns', None) is not None:
return_type = build_expr(ctx, py_def.returns)
decl = Decl(r, param_list, return_type)
is_method = self_name is not None
if type_line is not None:
type_comment_decl = torch._C.parse_type_comment(type_line)
decl = torch._C.merge_type_from_type_comment(decl, type_comment_decl, is_method)
return Def(Ident(r, def_name),
decl,
build_stmts(ctx, body))
_vararg_kwarg_err = ("Compiled functions can't take variable number of arguments "
"or use keyword-only arguments with defaults")
def build_param_list(ctx, py_args, self_name):
if py_args.kwarg is not None:
expr = py_args.kwarg
ctx_range = ctx.make_range(expr.lineno, expr.col_offset - 1, expr.col_offset + len(expr.arg))
raise NotSupportedError(ctx_range, _vararg_kwarg_err)
if py_args.vararg is not None:
expr = py_args.vararg
ctx_range = ctx.make_range(expr.lineno, expr.col_offset - 1, expr.col_offset + len(expr.arg))
raise NotSupportedError(ctx_range, _vararg_kwarg_err)
if len(py_args.kw_defaults) > 0:
# kw_defaults is a list of the values for the kwargs (which default to None),
# so they don't actually have line numbers.
for arg in py_args.kw_defaults:
if arg is not None:
ctx_range = build_expr(ctx, arg).range()
raise NotSupportedError(ctx_range, _vararg_kwarg_err)
result = [build_param(ctx, arg, self_name, False) for arg in py_args.args]
result += [build_param(ctx, arg, self_name, True) for arg in py_args.kwonlyargs]
return result
def build_param(ctx, py_arg, self_name, kwarg_only):
# NB: In Python3 py_arg is a pair of (str arg, expr? annotation)
name = py_arg.arg
r = ctx.make_range(py_arg.lineno, py_arg.col_offset, py_arg.col_offset + len(name))
if getattr(py_arg, 'annotation', None) is not None:
annotation_expr = build_expr(ctx, py_arg.annotation)
elif self_name is not None and name == 'self':
annotation_expr = Var(Ident(r, self_name))
else:
annotation_expr = EmptyTypeAnnotation(r)
return Param(annotation_expr, Ident(r, name), kwarg_only)
Loading ...