Skip to content

Magic Methods: Special Methods for ML Frameworks

Overview

Magic methods (dunder methods) enable operator overloading and framework magic: - __init__ = Constructor - __call__ = Make objects callable (how model(x) works) - __getitem__ = Indexing (batches, parameters) - __setitem__ = Assignment - +, -, *, @ = Arithmetic operators - __len__ = Length - __repr__ = Debugging representation

Understanding magic methods reveals how PyTorch enables intuitive syntax.

Core Pattern: How Python Calls Magic Methods

# Python translates operator syntax to magic method calls:

x + y  # Calls x.__add__(y)
x - y  # Calls x.__sub__(y)
x * y  # Calls x.__mul__(y)
x @ y  # Calls x.__matmul__(y)  # Matrix multiplication

x == y  # Calls x.__eq__(y)
x < y   # Calls x.__lt__(y)

len(x)  # Calls x.__len__()
x[0]    # Calls x.__getitem__(0)
x[0] = 5  # Calls x.__setitem__(0, 5)

bool(x)  # Calls x.__bool__()
str(x)   # Calls x.__str__()
repr(x)  # Calls x.__repr__()

PyTorch's Magic Methods

__init__: Initialization

import torch

class Linear(torch.nn.Module):
    def __init__(self, in_features, out_features):
        super().__init__()  # Calls torch.nn.Module.__init__
        self.weight = torch.nn.Parameter(
            torch.randn(out_features, in_features) / (in_features ** 0.5)
        )
        self.bias = torch.nn.Parameter(torch.zeros(out_features))

# __init__ called automatically
layer = Linear(10, 5)

__call__: Making Objects Callable

The most important magic method for PyTorch:

class Layer:
    def __call__(self, x):
        """Called when using layer(x) syntax."""
        print(f"Forward pass on input shape {x.shape}")
        return x * 2

layer = Layer()
output = layer(torch.randn(3, 10))  # Calls layer.__call__(...)

# PyTorch: This is how model(x) works
class Model(torch.nn.Module):
    def forward(self, x):
        # Note: forward is wrapped by nn.Module.__call__
        # nn.Module.__call__ handles registration hooks
        return self.fc(x)

model = Model(torch.nn.Linear(10, 5))
output = model(torch.randn(3, 10))  # Calls Model.__call__ → forward()

Inside PyTorch, nn.Module.call: 

# Simplified version of what nn.Module does
class Module:
    def __call__(self, *input, **kwargs):
        # Pre-forward hooks
        for hook in self._pre_forward_hooks:
            hook(self, input)

        # Actual forward
        output = self.forward(*input, **kwargs)

        # Post-forward hooks
        for hook in self._forward_hooks:
            hook(self, input, output)

        return output

__getitem__: Indexing and Slicing

class Batch:
    def __init__(self, tensors):
        self.tensors = tensors

    def __getitem__(self, index):
        """Support indexing like batch[0]."""
        return self.tensors[index]

batch = Batch(torch.randn(32, 3, 224, 224))
sample = batch[0]  # Calls batch.__getitem__(0)
print(sample.shape)  # torch.Size([3, 224, 224])

# Also enables slicing
subset = batch[0:10]  # Calls batch.__getitem__(slice(0, 10))

__setitem__: Assignment

class ParameterDict:
    def __init__(self):
        self.params = {}

    def __setitem__(self, key, value):
        """Support assignment like params['weight'] = tensor."""
        if isinstance(value, torch.nn.Parameter):
            self.params[key] = value
        else:
            raise TypeError("Must be a Parameter")

params = ParameterDict()
params['weight'] = torch.nn.Parameter(torch.randn(10, 5))

__len__: Length

class Dataset:
    def __init__(self, data):
        self.data = data

    def __len__(self):
        """Support len(dataset)."""
        return len(self.data)

dataset = Dataset([1, 2, 3, 4, 5])
print(len(dataset))  # 5

# PyTorch DataLoader uses this
print(len(torch_dataset))  # Queries __len__

Arithmetic Operators: Mathematical Operations

Basic Operations

class Tensor:
    def __init__(self, data):
        self.data = data

    # Addition
    def __add__(self, other):
        if isinstance(other, Tensor):
            return Tensor(self.data + other.data)
        return Tensor(self.data + other)

    # Subtraction
    def __sub__(self, other):
        if isinstance(other, Tensor):
            return Tensor(self.data - other.data)
        return Tensor(self.data - other)

    # Multiplication
    def __mul__(self, other):
        if isinstance(other, Tensor):
            return Tensor(self.data * other.data)
        return Tensor(self.data * other)

    # Matrix multiplication (crucial for neural nets)
    def __matmul__(self, other):
        return Tensor(self.data @ other.data)

# Usage (same as PyTorch)
t1 = Tensor([[1, 2], [3, 4]])
t2 = Tensor([[5, 6], [7, 8]])

result = t1 + t2       # Calls t1.__add__(t2)
result = t1 * 2        # Calls t1.__mul__(2)
result = t1 @ t2       # Calls t1.__matmul__(t2)

Right Operations: Handling Scalar + Tensor

class Tensor:
    def __add__(self, other):
        # Called for tensor + 5
        return self._add_impl(other)

    def __radd__(self, other):
        # Called for 5 + tensor (when other.__add__ doesn't know how)
        # This enables: 5 + tensor = tensor + 5
        return self.__add__(other)

    def __rmul__(self, other):
        # Called for 5 * tensor
        return self.__mul__(other)

    def __rmatmul__(self, other):
        # Called for other @ tensor
        return self.__matmul__(other)

# These enable natural syntax
t = Tensor([1, 2, 3])
result1 = t + 5      # t.__add__(5)
result2 = 5 + t      # t.__radd__(5) - works because of __radd__
result3 = 2 * t      # t.__rmul__(2)

In-place Operations

class Tensor:
    def __iadd__(self, other):
        """Called for tensor += other."""
        self.data += other
        return self  # Must return self

    def __imul__(self, other):
        """Called for tensor *= other."""
        self.data *= other
        return self

# Usage
t = Tensor([1, 2, 3])
t += 5  # Calls t.__iadd__(5), modifies in place
t *= 2  # Calls t.__imul__(2)

Comparison Operators

class Tensor:
    def __eq__(self, other):
        """Called for tensor1 == tensor2."""
        if isinstance(other, Tensor):
            return Tensor(self.data == other.data)
        return Tensor(self.data == other)

    def __lt__(self, other):
        """Called for tensor1 < tensor2."""
        if isinstance(other, Tensor):
            return Tensor(self.data < other.data)
        return Tensor(self.data < other)

    def __gt__(self, other):
        """Called for tensor1 > tensor2."""
        if isinstance(other, Tensor):
            return Tensor(self.data > other.data)
        return Tensor(self.data > other)

# PyTorch uses these for comparisons
t1 = Tensor([1, 5, 3])
t2 = Tensor([2, 4, 3])

mask = t1 < t2  # Creates boolean tensor
print(mask.data)  # [True, False, True]

String Representations: Debugging

__repr__ vs __str__

class Tensor:
    def __init__(self, data, dtype=torch.float32):
        self.data = data
        self.dtype = dtype
        self.shape = data.shape

    def __repr__(self):
        """Unambiguous representation (for debugging)."""
        # Should ideally be evaluable: eval(repr(x)) == x
        return f"Tensor({self.data.tolist()}, dtype={self.dtype})"

    def __str__(self):
        """Human-readable string (for printing)."""
        return f"Tensor of shape {self.shape}"

t = Tensor(torch.tensor([1, 2, 3]))
print(repr(t))  # Tensor([1, 2, 3], dtype=torch.float32)
print(str(t))   # Tensor of shape torch.Size([3])
print(t)        # Uses __str__ by default

PyTorch's Tensor Representation

# PyTorch tensors have nice __repr__
t = torch.tensor([1, 2, 3], dtype=torch.float32)
print(repr(t))
# tensor([1., 2., 3.])

# With more info
t = torch.randn(2, 3, requires_grad=True)
print(t)
# tensor([[...], [...]], requires_grad=True)

Numeric Conversions

__int__, __float__, __bool__

class Tensor:
    def __int__(self):
        """Convert to int (only works for scalar tensors)."""
        if self.data.size == 1:
            return int(self.data.flat[0])
        raise ValueError("Cannot convert multi-element tensor to int")

    def __float__(self):
        """Convert to float."""
        if self.data.size == 1:
            return float(self.data.flat[0])
        raise ValueError("Cannot convert multi-element tensor to float")

    def __bool__(self):
        """Convert to bool (for if statements)."""
        if self.data.size == 1:
            return bool(self.data.flat[0])
        raise RuntimeError("Boolean value of tensor with >1 element is ambiguous")

# Usage (PyTorch pattern)
t = torch.tensor(5)
x = int(t)  # 5
y = float(t)  # 5.0
if torch.tensor(True):  # Works
    print("True tensor")

if torch.tensor(False):  # Works
    print("False tensor")

if torch.tensor([1, 2]):  # RuntimeError - ambiguous!
    pass

Context Managers: __enter__ and __exit__

class DeviceContext:
    """Move tensors to device temporarily."""
    def __init__(self, device):
        self.device = device

    def __enter__(self):
        """Called when entering with block."""
        torch.cuda.set_device(self.device)
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        """Called when exiting with block."""
        torch.cuda.set_device(0)  # Reset
        return False

# Usage
with DeviceContext(1):
    model.to("cuda:1")
    output = model(input_data)

Practical PyTorch Example

Custom Tensor Class

class CustomTensor:
    def __init__(self, data):
        self.data = torch.tensor(data)
        self.requires_grad = False

    # Arithmetic
    def __add__(self, other):
        if isinstance(other, CustomTensor):
            return CustomTensor(self.data + other.data)
        return CustomTensor(self.data + other)

    def __matmul__(self, other):
        if isinstance(other, CustomTensor):
            return CustomTensor(self.data @ other.data)
        return CustomTensor(self.data @ other)

    # Indexing
    def __getitem__(self, index):
        return CustomTensor(self.data[index])

    def __len__(self):
        return len(self.data)

    # Representation
    def __repr__(self):
        return f"CustomTensor({self.data})"

    def __str__(self):
        return f"CustomTensor of shape {self.data.shape}"

# Full usage
t1 = CustomTensor([[1, 2], [3, 4]])
t2 = CustomTensor([[5, 6], [7, 8]])

result = t1 @ t2       # Matrix multiply
print(result)          # __str__ called
print(result[0])       # __getitem__ called
print(len(result))     # __len__ called

Common Pitfalls

Pitfall 1: Forgetting to Return Self in In-Place Ops

class BadTensor:
    def __iadd__(self, other):
        self.data += other
        # WRONG: No return statement!
        # Returns None, so t += 5 makes t = None

class GoodTensor:
    def __iadd__(self, other):
        self.data += other
        return self  # CORRECT: Return self

Pitfall 2: Ambiguous Boolean Evaluation

# PyTorch prevents this:
if torch.tensor([1, 2]):  # RuntimeError
    pass

# PyTorch tensors implement __bool__ to raise error
# Prevents accidental Python control flow bugs

Pitfall 3: Not Handling Type Mismatches

class BadTensor:
    def __add__(self, other):
        # Crashes if other is not CustomTensor
        return BadTensor(self.data + other.data)

class GoodTensor:
    def __add__(self, other):
        if isinstance(other, GoodTensor):
            return GoodTensor(self.data + other.data)
        else:
            # Try to work with scalars
            return GoodTensor(self.data + other)

Summary

Magic Method Use Case Example
__init__ Constructor t = Tensor(data)
__call__ Make callable output = model(x)
__getitem__ Indexing batch[0]
__setitem__ Assignment batch[0] = value
__len__ Length len(dataset)
__add__ Addition t1 + t2
__mul__ Multiplication t * 2
__matmul__ Matrix mult t1 @ t2
__radd__ Right add 5 + tensor
__iadd__ In-place add t += 5
__eq__ Equality t1 == t2
__lt__ Less than t1 < t2
__repr__ Debug repr repr(t)
__str__ Print string str(t)
__bool__ Bool conv bool(t)
__enter__ Context enter with x:
__exit__ Context exit with x: