Magic Methods

Overview

Magic methods (dunder methods) enable operator overloading and framework magic:

• __init__ = Constructor
• __call__ = Make objects callable (how model(x) works)
• __getitem__ = Indexing (batches, parameters)
• __setitem__ = Assignment
• +, -, *, @ = Arithmetic operators
• __len__ = Length
• __repr__ = Debugging representation

Understanding magic methods reveals how PyTorch enables intuitive syntax.

---

Core Pattern: How Python Calls Magic Methods

# Python translates operator syntax to magic method calls:

x + y # Calls x.__add__(y)
x - y # Calls x.__sub__(y)
x * y # Calls x.__mul__(y)
x @ y # Calls x.__matmul__(y) # Matrix multiplication

x == y # Calls x.__eq__(y)
x < y # Calls x.__lt__(y)

len(x) # Calls x.__len__()
x[0] # Calls x.__getitem__(0)
x[0] = 5 # Calls x.__setitem__(0, 5)

bool(x) # Calls x.__bool__()
str(x) # Calls x.__str__()
repr(x) # Calls x.__repr__()

-

PyTorch's Magic Methods

__init__: Initialization

import torch

class Linear(torch.nn.Module):
 def __init__(self, in_features, out_features):
 super().__init__() # Calls torch.nn.Module.__init__
 self.weight = torch.nn.Parameter(
 torch.randn(out_features, in_features) / (in_features ** 0.5)
)
 self.bias = torch.nn.Parameter(torch.zeros(out_features))

# __init__ called automatically
layer = Linear(10, 5)

__call__: Making Objects Callable

The most important magic method for PyTorch:

class Layer:
 def __call__(self, x):
 """Called when using layer(x) syntax."""
 print(f"Forward pass on input shape {x.shape}")
 return x * 2

layer = Layer()
output = layer(torch.randn(3, 10)) # Calls layer.__call__(...)

# PyTorch
class Model(torch.nn.Module):
 def forward(self, x):
 # Note: forward is wrapped by nn.Module.__call__
 # nn.Module.__call__ handles registration hooks
 return self.fc(x)

model = Model(torch.nn.Linear(10, 5))
output = model(torch.randn(3, 10)) # Calls Model.__call__ → forward()

Inside PyTorch, nn.Module.call:

# Simplified version of what nn.Module does
class Module:
 def __call__(self, *input, **kwargs):
 # Pre-forward hooks
 for hook in self._pre_forward_hooks:
 hook(self, input)

 # Actual forward
 output = self.forward(*input, **kwargs)

 # Post-forward hooks
 for hook in self._forward_hooks:
 hook(self, input, output)

 return output

__getitem__: Indexing and Slicing

class Batch:
 def __init__(self, tensors):
 self.tensors = tensors

 def __getitem__(self, index):
 """Support indexing like batch[0]."""
 return self.tensors[index]

batch = Batch(torch.randn(32, 3, 224, 224))
sample = batch[0] # Calls batch.__getitem__(0)
print(sample.shape) # torch.Size([3, 224, 224])

# Also enables slicing
subset = batch[0:10] # Calls batch.__getitem__(slice(0, 10))

__setitem__: Assignment

class ParameterDict:
 def __init__(self):
 self.params = {}

 def __setitem__(self, key, value):
 """Support assignment like params['weight'] = tensor."""
 if isinstance(value, torch.nn.Parameter):
 self.params[key] = value
 else:
 raise TypeError("Must be a Parameter")

params = ParameterDict()
params['weight'] = torch.nn.Parameter(torch.randn(10, 5))

__len__: Length

class Dataset:
 def __init__(self, data):
 self.data = data

 def __len__(self):
 """Support len(dataset)."""
 return len(self.data)

dataset = Dataset([1, 2, 3, 4, 5])
print(len(dataset)) # 5

# PyTorch DataLoader uses this
print(len(torch_dataset)) # Queries __len__

---

Arithmetic Operators: Mathematical Operations

Basic Operations

class Tensor:
 def __init__(self, data):
 self.data = data

 # Addition
 def __add__(self, other):
 if isinstance(other, Tensor):
 return Tensor(self.data + other.data)
 return Tensor(self.data + other)

 # Subtraction
 def __sub__(self, other):
 if isinstance(other, Tensor):
 return Tensor(self.data - other.data)
 return Tensor(self.data - other)

 # Multiplication
 def __mul__(self, other):
 if isinstance(other, Tensor):
 return Tensor(self.data * other.data)
 return Tensor(self.data * other)

 # Matrix multiplication (crucial for neural nets)
 def __matmul__(self, other):
 return Tensor(self.data @ other.data)

# Usage (same as PyTorch)
t1 = Tensor([[1, 2], [3, 4]])
t2 = Tensor([[5, 6], [7, 8]])

result = t1 + t2 # Calls t1.__add__(t2)
result = t1 * 2 # Calls t1.__mul__(2)
result = t1 @ t2 # Calls t1.__matmul__(t2)

Right Operations: Handling Scalar + Tensor

class Tensor:
 def __add__(self, other):
 # Called for tensor + 5
 return self._add_impl(other)

 def __radd__(self, other):
 # Called for 5 + tensor (when other.__add__ doesn't know how)
 # This enables: 5 + tensor = tensor + 5
 return self.__add__(other)

 def __rmul__(self, other):
 # Called for 5 * tensor
 return self.__mul__(other)

 def __rmatmul__(self, other):
 # Called for other @ tensor
 return self.__matmul__(other)

# These enable natural syntax
t = Tensor([1, 2, 3])
result1 = t + 5 # t.__add__(5)
result2 = 5 + t # t.__radd__(5) - works because of __radd__
result3 = 2 * t # t.__rmul__(2)

In-place Operations

class Tensor:
 def __iadd__(self, other):
 """Called for tensor += other."""
 self.data += other
 return self # Must return self

 def __imul__(self, other):
 """Called for tensor *= other."""
 self.data *= other
 return self

# Usage
t = Tensor([1, 2, 3])
t += 5 # Calls t.__iadd__(5), modifies in place
t *= 2 # Calls t.__imul__(2)

Comparison Operators

class Tensor:
 def __eq__(self, other):
 """Called for tensor1 == tensor2."""
 if isinstance(other, Tensor):
 return Tensor(self.data == other.data)
 return Tensor(self.data == other)

 def __lt__(self, other):
 """Called for tensor1 < tensor2."""
 if isinstance(other, Tensor):
 return Tensor(self.data < other.data)
 return Tensor(self.data < other)

 def __gt__(self, other):
 """Called for tensor1 > tensor2."""
 if isinstance(other, Tensor):
 return Tensor(self.data > other.data)
 return Tensor(self.data > other)

# PyTorch uses these for comparisons
t1 = Tensor([1, 5, 3])
t2 = Tensor([2, 4, 3])

mask = t1 < t2 # Creates boolean tensor
print(mask.data) # [True, False, True]

---

String Representations: Debugging

__repr__ vs __str__

class Tensor:
 def __init__(self, data, dtype=torch.float32):
 self.data = data
 self.dtype = dtype
 self.shape = data.shape

 def __repr__(self):
 """Unambiguous representation (for debugging)."""
 # Should ideally be evaluable: eval(repr(x)) == x
 return f"Tensor({self.data.tolist()}, dtype={self.dtype})"

 def __str__(self):
 """Human-readable string (for printing)."""
 return f"Tensor of shape {self.shape}"

t = Tensor(torch.tensor([1, 2, 3]))
print(repr(t)) # Tensor([1, 2, 3], dtype=torch.float32)
print(str(t)) # Tensor of shape torch.Size([3])
print(t) # Uses __str__ by default

PyTorch's Tensor Representation

# PyTorch tensors have nice __repr__
t = torch.tensor([1, 2, 3], dtype=torch.float32)
print(repr(t))
# tensor([1., 2., 3.])

# With more info
t = torch.randn(2, 3, requires_grad=True)
print(t)
# tensor([[...], [...]], requires_grad=True)

---

Numeric Conversions

__int__, __float__, __bool__

class Tensor:
 def __int__(self):
 """Convert to int (only works for scalar tensors)."""
 if self.data.size == 1:
 return int(self.data.flat[0])
 raise ValueError("Cannot convert multi-element tensor to int")

 def __float__(self):
 """Convert to float."""
 if self.data.size == 1:
 return float(self.data.flat[0])
 raise ValueError("Cannot convert multi-element tensor to float")

 def __bool__(self):
 """Convert to bool (for if statements)."""
 if self.data.size == 1:
 return bool(self.data.flat[0])
 raise RuntimeError("Boolean value of tensor with >1 element is ambiguous")

# Usage (PyTorch pattern)
t = torch.tensor(5)
x = int(t) # 5
y = float(t) # 5.0
if torch.tensor(True): # Works
 print("True tensor")

if torch.tensor(False): # Works
 print("False tensor")

if torch.tensor([1, 2]): # RuntimeError - ambiguous!
 pass

---

Context Managers: __enter__ and __exit__

class DeviceContext:
 """Move tensors to device temporarily."""
 def __init__(self, device):
 self.device = device

 def __enter__(self):
 """Called when entering with block."""
 torch.cuda.set_device(self.device)
 return self

 def __exit__(self, exc_type, exc_val, exc_tb):
 """Called when exiting with block."""
 torch.cuda.set_device(0) # Reset
 return False

# Usage
with DeviceContext(1):
 model.to("cuda:1")
 output = model(input_data)

---

Practical PyTorch Example

Custom Tensor Class

class CustomTensor:
 def __init__(self, data):
 self.data = torch.tensor(data)
 self.requires_grad = False

 # Arithmetic
 def __add__(self, other):
 if isinstance(other, CustomTensor):
 return CustomTensor(self.data + other.data)
 return CustomTensor(self.data + other)

 def __matmul__(self, other):
 if isinstance(other, CustomTensor):
 return CustomTensor(self.data @ other.data)
 return CustomTensor(self.data @ other)

 # Indexing
 def __getitem__(self, index):
 return CustomTensor(self.data[index])

 def __len__(self):
 return len(self.data)

 # Representation
 def __repr__(self):
 return f"CustomTensor({self.data})"

 def __str__(self):
 return f"CustomTensor of shape {self.data.shape}"

# Full usage
t1 = CustomTensor([[1, 2], [3, 4]])
t2 = CustomTensor([[5, 6], [7, 8]])

result = t1 @ t2 # Matrix multiply
print(result) # __str__ called
print(result[0]) # __getitem__ called
print(len(result)) # __len__ called

---

Common Pitfalls

Pitfall 1: Forgetting to Return Self in In-Place Ops

class BadTensor:
 def __iadd__(self, other):
 self.data += other
 # WRONG: No return statement!
 # Returns None, so t += 5 makes t = None

class GoodTensor:
 def __iadd__(self, other):
 self.data += other
 return self # CORRECT: Return self

Pitfall 2: Ambiguous Boolean Evaluation

# PyTorch prevents this:
if torch.tensor([1, 2]): # RuntimeError
 pass

# PyTorch tensors implement __bool__ to raise error
# Prevents accidental Python control flow bugs

Pitfall 3: Not Handling Type Mismatches

class BadTensor:
 def __add__(self, other):
 # Crashes if other is not CustomTensor
 return BadTensor(self.data + other.data)

class GoodTensor:
 def __add__(self, other):
 if isinstance(other, GoodTensor):
 return GoodTensor(self.data + other.data)
 else:
 # Try to work with scalars
 return GoodTensor(self.data + other)

---

Summary

Magic Method	Use Case	Example
__init__	Constructor	t = Tensor(data)
__call__	Make callable	output = model(x)
__getitem__	Indexing	batch[0]
__setitem__	Assignment	batch[0] = value
__len__	Length	len(dataset)
__add__	Addition	t1 + t2
__mul__	Multiplication	t * 2
__matmul__	Matrix mult	t1 @ t2
__radd__	Right add	5 + tensor
__iadd__	In-place add	t += 5
__eq__	Equality	t1 == t2
__lt__	Less than	t1 < t2
__repr__	Debug repr	repr(t)
__str__	Print string	str(t)
__bool__	Bool conv	bool(t)
__enter__	Context enter	with x:
__exit__	Context exit	with x:

-

Related Topics

• [01 Classes & Inheritance](/05-py3/02-object-oriented-patterns/(01-classes-inheritance/) - Classes that use magic methods
• [04 Descriptors & Properties](/05-py3/02-object-oriented-patterns/(04-descriptors-properties/) - Advanced attribute management
• 00 Readme - Operator overloading alternatives
• 00 Readme - Tensor implementations