Global Interpreter Lock (GIL): Python's Threading Limitation

Overview

The GIL is Python's most misunderstood feature: - Prevents true parallelism: Only one thread executes Python code at a time - Released during I/O: Network, disk operations release the GIL - Released by native code: NumPy, PyTorch, C extensions release the GIL - Implications for ML: Pure Python code can't parallelize, but NumPy/PyTorch can

Understanding the GIL is critical for building concurrent systems.

---

What is the GIL?

The Problem GIL Solves

Python uses reference counting for memory management:

# Each object has a reference count
x = []  # ref_count = 1
y = x   # ref_count = 2
del x   # ref_count = 1
del y   # ref_count = 0, object freed

# Reference count is NOT thread-safe
# Thread 1: decrements ref_count
# Thread 2: decrements ref_count
# Race condition: ref_count goes to 0 too early or too late!

The GIL ensures reference counting is thread-safe by allowing only one thread to execute Python code at a time.

GIL Mechanics

import threading
import time

counter = 0
lock = threading.Lock()  # Simulating GIL

def increment_with_gil():
    """Thread can only run when holding GIL."""
    global counter

    # Acquire GIL (simplified)
    with lock:  # Can't proceed without lock (GIL)
        temp = counter
        # ... other thread could acquire GIL here in real Python
        counter = temp + 1
    # Release GIL

# Two threads trying to increment
threads = []
for _ in range(2):
    t = threading.Thread(target=increment_with_gil)
    threads.append(t)
    t.start()

for t in threads:
    t.join()

print(counter)  # Correct result (with proper locking)

When GIL is Released

import threading
import time

# GIL released during I/O
def network_io():
    """GIL released during network operations."""
    # Other threads can run while this waits for network
    time.sleep(1)  # Simulates I/O (GIL released)
    print("Network done")

# GIL released by native code
import numpy as np

def numpy_compute():
    """GIL released during NumPy operations."""
    # Other threads can run while NumPy computes
    arr = np.random.randn(1000, 1000)
    result = np.sum(arr)
    print(f"NumPy result: {result}")

# Both can run in parallel!
t1 = threading.Thread(target=network_io)
t2 = threading.Thread(target=numpy_compute)

t1.start()
t2.start()

t1.join()
t2.join()

# Both finished in ~1 second (parallel)
# Without GIL release, would take ~2 seconds (serial)

---

GIL Impact on Threading

Demonstrate GIL Contention

import threading
import time

# CPU-bound work (GIL HELD)
def cpu_bound(n):
    """Pure Python computation - GIL prevents parallelism."""
    total = 0
    for i in range(n):
        total += i
    return total

# Single-threaded
start = time.time()
result1 = cpu_bound(100000000)
result2 = cpu_bound(100000000)
single_time = time.time() - start
print(f"Single-threaded: {single_time:.2f}s")

# Multi-threaded (SLOWER!)
start = time.time()
t1 = threading.Thread(target=cpu_bound, args=(100000000,))
t2 = threading.Thread(target=cpu_bound, args=(100000000,))
t1.start()
t2.start()
t1.join()
t2.join()
multi_time = time.time() - start
print(f"Multi-threaded: {multi_time:.2f}s")

print(f"Slowdown: {multi_time / single_time:.1f}x")
# Result: Multi-threaded is SLOWER due to GIL contention!

I/O-Bound Work Benefits from Threading

import threading
import time

# I/O-bound work (GIL RELEASED)
def io_bound(n):
    """Simulate I/O - GIL is released."""
    for i in range(n):
        time.sleep(0.001)  # Simulate I/O, GIL released

# Single-threaded: 5 seconds
start = time.time()
io_bound(5)
single_time = time.time() - start
print(f"Single-threaded I/O: {single_time:.2f}s")

# Multi-threaded: ~1 second (all 5 threads run in parallel)
start = time.time()
threads = []
for _ in range(5):
    t = threading.Thread(target=io_bound, args=(5,))
    threads.append(t)
    t.start()

for t in threads:
    t.join()
multi_time = time.time() - start
print(f"Multi-threaded I/O: {multi_time:.2f}s")

print(f"Speedup: {single_time / multi_time:.1f}x")
# Result: 5x speedup with threading!

---

GIL in ML Systems

NumPy Releases GIL

import threading
import numpy as np
import time

def numpy_compute(name):
    """NumPy operations release GIL."""
    print(f"{name} starting...")

    # GIL released during NumPy computation
    arr = np.random.randn(2000, 2000)
    result = np.dot(arr, arr)

    print(f"{name} done: {result.shape}")

# Two threads can compute in parallel with NumPy
start = time.time()

t1 = threading.Thread(target=numpy_compute, args=("Thread 1",))
t2 = threading.Thread(target=numpy_compute, args=("Thread 2",))

t1.start()
t2.start()

t1.join()
t2.join()

elapsed = time.time() - start
print(f"Parallel time: {elapsed:.2f}s")

# Sequential for comparison
start = time.time()
numpy_compute("Sequential 1")
numpy_compute("Sequential 2")
sequential_time = time.time() - start
print(f"Sequential time: {sequential_time:.2f}s")

print(f"Speedup: {sequential_time / elapsed:.1f}x")
# Result: ~1.8-2x speedup (not quite 2x due to overhead)

PyTorch Releases GIL

import threading
import torch
import time

def torch_compute(name, device='cpu'):
    """PyTorch operations release GIL."""
    print(f"{name} starting...")

    # GIL released during PyTorch computation
    x = torch.randn(2000, 2000, device=device)
    y = torch.randn(2000, 2000, device=device)
    result = torch.matmul(x, y)

    print(f"{name} done: {result.shape}")

# CPU compute (GIL released)
start = time.time()

t1 = threading.Thread(target=torch_compute, args=("Thread 1", "cpu"))
t2 = threading.Thread(target=torch_compute, args=("Thread 2", "cpu"))

t1.start()
t2.start()

t1.join()
t2.join()

cpu_time = time.time() - start
print(f"Parallel CPU time: {cpu_time:.2f}s")

# GPU compute (no GIL even relevant, computation on GPU)
start = time.time()

t1 = threading.Thread(target=torch_compute, args=("Thread 1", "cuda"))
t2 = threading.Thread(target=torch_compute, args=("Thread 2", "cuda"))

t1.start()
t2.start()

t1.join()
t2.join()

gpu_time = time.time() - start
print(f"Parallel GPU time: {gpu_time:.2f}s")

---

Strategies for GIL Avoidance

Strategy 1: Multiprocessing Instead of Threading

from multiprocessing import Process
import time

def cpu_bound_task(n):
    """Pure Python CPU-bound work."""
    total = 0
    for i in range(n):
        total += i
    return total

# Single process
start = time.time()
result1 = cpu_bound_task(100000000)
result2 = cpu_bound_task(100000000)
single_time = time.time() - start

# Multi-process (true parallelism, no GIL!)
start = time.time()
p1 = Process(target=cpu_bound_task, args=(100000000,))
p2 = Process(target=cpu_bound_task, args=(100000000,))
p1.start()
p2.start()
p1.join()
p2.join()
multi_time = time.time() - start

print(f"Single-process: {single_time:.2f}s")
print(f"Multi-process: {multi_time:.2f}s")
print(f"Speedup: {single_time / multi_time:.1f}x")
# Result: ~2x speedup with 2 processes!

Strategy 2: Use NumPy/PyTorch for Heavy Lifting

import numpy as np
import threading
import time

# SLOW: Pure Python with threading (GIL contention)
def slow_array_sum():
    """Pure Python - GIL prevents parallelism."""
    data = list(range(10000000))
    total = 0
    for x in data:
        total += x
    return total

# FAST: NumPy (releases GIL, parallelizes internally)
def fast_array_sum():
    """NumPy - releases GIL, uses SIMD."""
    data = np.arange(10000000)
    return np.sum(data)

# Benchmark
start = time.time()
for _ in range(10):
    slow_array_sum()
slow_time = time.time() - start

start = time.time()
for _ in range(10):
    fast_array_sum()
fast_time = time.time() - start

print(f"Pure Python: {slow_time:.2f}s")
print(f"NumPy: {fast_time:.2f}s")
print(f"Speedup: {slow_time / fast_time:.0f}x")
# Typical: 100-1000x faster with NumPy!

---

Inference Server Pattern

Single-Threaded (Good for GPU)

import torch
import threading

class GPUInferenceServer:
    """Single-threaded GPU inference (simplest)."""

    def __init__(self, model_path):
        self.model = torch.load(model_path)
        self.model = self.model.cuda().eval()
        self.lock = threading.Lock()  # Serialize GPU access

    def infer(self, request):
        """Serialize all GPU access."""
        with self.lock:  # Only one inference at a time
            x = torch.tensor(request, device='cuda')
            with torch.no_grad():
                return self.model(x).cpu().numpy()

# All requests wait for lock (sequential inference)

Multi-Threaded (Good for CPU or I/O)

class CPUInferenceServer:
    """Multi-threaded CPU inference."""

    def __init__(self, model_path):
        self.model = torch.load(model_path)
        self.model = self.model.cpu().eval()
        # No lock needed - PyTorch releases GIL during compute

    def infer(self, request):
        """No locking needed - PyTorch releases GIL."""
        x = torch.tensor(request)
        with torch.no_grad():
            return self.model(x).numpy()

# Threads can infer in parallel!

Async (Good for High Concurrency)

import asyncio
import torch

class AsyncInferenceServer:
    """Async inference for high concurrency."""

    def __init__(self, model_path):
        self.model = torch.load(model_path)
        self.model = self.model.cpu().eval()

    async def infer_async(self, request):
        """Async inference - submit and wait."""
        # Move to thread pool (release GIL, don't block event loop)
        loop = asyncio.get_event_loop()
        x = torch.tensor(request)

        # Run in thread pool
        result = await loop.run_in_executor(
            None,  # Use default thread pool
            lambda: self._infer_impl(x)
        )
        return result

    def _infer_impl(self, x):
        """Actual inference (runs in thread, GIL released by PyTorch)."""
        with torch.no_grad():
            return self.model(x).numpy()

# Can handle 1000s of concurrent requests!

---

GIL Behavior Visualization

# Timeline of GIL with threading

# Pure Python (CPU-bound)
# Time: 0-1s
# Thread 1: ████████████ (GIL held entire time)
# Thread 2: waiting, waiting, waiting, ████████████
# Total: 2 seconds

# NumPy (I/O-bound with threading)
# Time: 0-1s
# Thread 1: ██░░░░████░░░░
# Thread 2: ░░░░████░░░░██
# Total: ~1 second (parallelism!)

# Real parallel (Multiprocessing)
# Process 1: ████████████ (CPU 0)
# Process 2: ████████████ (CPU 1)
# Total: 1 second (true parallelism!)

---

Summary: GIL Decision Tree

Is your code CPU-bound pure Python?
  - YES → Use multiprocessing
  - NO → Is it I/O-bound?
  - YES → Use threading (GIL released during I/O)
  - NO → Is it NumPy/PyTorch?
  - YES → Use threading (GIL released by native code)
  - NO → Use multiprocessing (CPU-bound)

---

Related Topics

• 04 Multithreading Vs Multiprocessing - Multiprocessing alternative
• 05 Async Await - Async alternative to threading
• 02 Memory Layout & Cache Efficiency - Performance tuning