CodeReading – 1. PyTorch

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/tensor.py#L66

new_storage = self.storage().__deepcopy__(memo)
...

new_tensor = self.new()
new_tensor.set_(new_storage, self.storage_offset(), self.size(), self.stride())
new_tensor.requires_grad = self.requires_grad

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/tensor.py#L35

class Tensor(torch._C._TensorBase):
    ...

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/tensor.py#L155

def backward(self, gradient=None, retain_graph=None, create_graph=False, inputs=None):
    r"""Computes the gradient of current tensor w.r.t. graph leaves.
    
        The graph is differentiated using the chain rule.
    ...
    torch.autograd.backward(self, gradient, retain_graph, create_graph, inputs=inputs)

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/tensor.py#L725

    @property
    def grad(self):
        """
        This attribute is ``None`` by default and becomes a Tensor the first time a call to
        :func:`backward` computes gradients for ``self``.
        The attribute will then contain the gradients computed and future calls to
        :func:`backward` will accumulate (add) gradients into it.
        """
        ...
        return self._grad

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/tensor.py#L187

def register_hook(self, hook):
    r"""Registers a backward hook.

    The hook will be called every time a gradient with respect to the
    Tensor is computed. The hook should have the following signature::
      hook(grad) -> Tensor or None
    ...

    Example::
      >>> v = torch.tensor([0., 0., 0.], requires_grad=True)
      >>> h = v.register_hook(lambda grad: grad * 2)  # double the gradient
      >>> v.backward(torch.tensor([1., 2., 3.]))
      >>> v.grad
       2
       4
       6
      [torch.FloatTensor of size (3,)]
      >>> h.remove()  # removes the hook

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/autograd/__init__.py#L57

def backward(
    tensors: _TensorOrTensors,
    grad_tensors: Optional[_TensorOrTensors] = None,
    retain_graph: Optional[bool] = None,
    create_graph: bool = False,
    grad_variables: Optional[_TensorOrTensors] = None,
) -> None:
    r"""Computes the sum of gradients of given tensors w.r.t. graph leaves.
    ...
    """

    ...
    Variable._execution_engine.run_backward(
        tensors, grad_tensors, retain_graph, create_graph,
        allow_unreachable=True)  # allow_unreachable flag

# Inherit from Function
class LinearFunction(Function):

    # Note that both forward and backward are @staticmethods
    @staticmethod
    # bias is an optional argument
    def forward(ctx, input, weight, bias=None):
        ctx.save_for_backward(input, weight, bias)
        output = input.mm(weight.t())
        if bias is not None:
            output += bias.unsqueeze(0).expand_as(output)
        return output

    # This function has only a single output, so it gets only one gradient
    @staticmethod
    def backward(ctx, grad_output):
        # This is a pattern that is very convenient - at the top of backward
        # unpack saved_tensors and initialize all gradients w.r.t. inputs to
        # None. Thanks to the fact that additional trailing Nones are
        # ignored, the return statement is simple even when the function has
        # optional inputs.
        input, weight, bias = ctx.saved_tensors
        grad_input = grad_weight = grad_bias = None

        # These needs_input_grad checks are optional and there only to
        # improve efficiency. If you want to make your code simpler, you can
        # skip them. Returning gradients for inputs that don't require it is
        # not an error.
        if ctx.needs_input_grad[0]:
            grad_input = grad_output.mm(weight)
        if ctx.needs_input_grad[1]:
            grad_weight = grad_output.t().mm(input)
        if bias is not None and ctx.needs_input_grad[2]:
            grad_bias = grad_output.sum(0)

        return grad_input, grad_weight, grad_bias

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/autograd/function.py#L110

class Function(with_metaclass(FunctionMeta, _C._FunctionBase, _ContextMethodMixin, _HookMixin)):
    r"""Records operation history and defines formulas for differentiating ops.
    ...

# https://github.com/pytorch/pytorch/blob/v1.6.0/torch/_six.py#L39

def with_metaclass(meta, *bases):
    """Create a base class with a metaclass."""
    # This requires a bit of explanation: the basic idea is to make a dummy
    # metaclass for one level of class instantiation that replaces itself with
    # the actual metaclass.
    class metaclass(meta):

        def __new__(cls, name, this_bases, d):
            return meta(name, bases, d)
    return type.__new__(metaclass, 'temporary_class', (), {})

# code : https://github.com/pytorch/pytorch/blob/v1.6.0/torch/autograd/function.py

class BackwardCFunction(_C._FunctionBase, _ContextMethodMixin, _HookMixin):
    _is_legacy = False

    def apply(self, *args):
        return self._forward_cls.backward(self, *args)

class FunctionMeta(type):
    """Function metaclass.
    ...
    """

    def __init__(cls, name, bases, attrs):
       ...

       backward_fn = type(name + 'Backward', (BackwardCFunction,), {'_forward_cls': cls})
       cls._backward_cls = backward_fn

class BackwardCFunction(_C._FunctionBase, _ContextMethodMixin, _HookMixin):
   ...

# BackwardCFunction.mro() : 해당 객체의 메서드 실행 순서를 표현합니다. (mro -> Method Resolution Order)
# [__main__.BackwardCFunction, __main__._C._FunctionBase, __main__._ContextMethodMixin, __main__._HookMixin, object]

class Function(with_metaclass(FunctionMeta, _C._FunctionBase, _ContextMethodMixin, _HookMixin)):
    ...

    # for the tracer
    is_traceable = False

    @staticmethod
    def forward(ctx: Any, *args: Any, **kwargs: Any) -> Any:
        raise NotImplementedError("You must implement the forward function for custom"
                                  " autograd.Function.")

    @staticmethod
    def backward(ctx: Any, *grad_outputs: Any) -> Any:
        raise NotImplementedError("You must implement the backward function for custom"
                                  " autograd.Function.")

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/nn/modules/module.py#L169

import torch.nn as nn
import torch.nn.functional as F

class Model(nn.Module):
	  def __init__(self):
	      super(Model, self).__init__()
	      self.conv1 = nn.Conv2d(1, 20, 5)
	      self.conv2 = nn.Conv2d(20, 20, 5)

	  def forward(self, x):
	      x = F.relu(self.conv1(x))
	      return F.relu(self.conv2(x)

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/nn/modules/module.py#L774

def __setattr__(self, name: str, value: Union[Tensor, 'Module']) -> None:
    if isinstance(value, Parameter):
        ...
    elif params is not None and name in params:
        ...
    else:
        modules = self.__dict__.get('_modules')
        if isinstance(value, Module):
            ...
        elif modules is not None and name in modules:
            ...
        else:
            buffers = self.__dict__.get('_buffers')

model = Model()
ouptuts = model(inputs)  # Model 의 __call__ 로 연결

# code: https://github.com/pytorch/pytorch/blob/v1.6.0/torch/nn/modules/module.py#L710

def _call_impl(self, *input, **kwargs):
    # 1. Forward Pre-hook
    for hook in itertools.chain(
            _global_forward_pre_hooks.values(),
            self._forward_pre_hooks.values()):
        result = hook(self, input)
        if result is not None:
            if not isinstance(result, tuple):
                result = (result,)
            input = result

    # 2. Forward
    if torch._C._get_tracing_state():
        result = self._slow_forward(*input, **kwargs)
    else:
        result = self.forward(*input, **kwargs)  # 우리가 정의하는 forward

    # 3. Forward Hook
    for hook in itertools.chain(
            _global_forward_hooks.values(),
            self._forward_hooks.values()):
        hook_result = hook(self, input, result)
        if hook_result is not None:
            result = hook_result

    # 4. Backward Hook 등록
    if (len(self._backward_hooks) > 0) or (len(_global_backward_hooks) > 0):
        var = result
        while not isinstance(var, torch.Tensor):
            if isinstance(var, dict):
                var = next((v for v in var.values() if isinstance(v, torch.Tensor)))
            else:
                var = var[0]
        grad_fn = var.grad_fn
        if grad_fn is not None:
            for hook in itertools.chain(
                    _global_backward_hooks.values(),
                    self._backward_hooks.values()):
                wrapper = functools.partial(hook, self)
                functools.update_wrapper(wrapper, hook)
                grad_fn.register_hook(wrapper)
    return result

__call__ : Callable[..., Any] = _call_impl

    @overload
    def to(self: T, device: Optional[Union[int, device]] = ..., dtype: Optional[Union[dtype, str]] = ...,
           non_blocking: bool = ...) -> T:
        ...

    @overload
    def to(self: T, dtype: Union[dtype, str], non_blocking: bool = ...) -> T:
        ...

    @overload
    def to(self: T, tensor: Tensor, non_blocking: bool = ...) -> T:
        ...

    def to(self, *args, **kwargs):

        device, dtype, non_blocking, convert_to_format = torch._C._nn._parse_to(*args, **kwargs)

        if dtype is not None:
            if not dtype.is_floating_point:
                raise TypeError('nn.Module.to only accepts floating point '
                                'dtypes, but got desired dtype={}'.format(dtype))

        def convert(t):
            if convert_to_format is not None and t.dim() == 4:
                return t.to(device, dtype if t.is_floating_point() else None, non_blocking, memory_format=convert_to_format)
            return t.to(device, dtype if t.is_floating_point() else None, non_blocking)

        return self._apply(convert)