JSONL Best Practices

Graceful Degradation

Handle parse errors without crashing the entire pipeline. Log errors and continue processing.

# Python: Skip malformed lines, log errors
import json
import logging

logging.basicConfig(level=logging.ERROR)

processed_count = 0
error_count = 0

with open('data.jsonl', 'r') as f:
    for line_num, line in enumerate(f, 1):
        try:
            obj = json.loads(line)
            process(obj)
            processed_count += 1
        except json.JSONDecodeError as e:
            logging.error(f"Line {line_num}: {e} | Content: {line[:100]}")
            error_count += 1
        except Exception as e:
            logging.error(f"Line {line_num}: Processing error: {e}")
            error_count += 1

print(f"Processed: {processed_count}, Errors: {error_count}")

Dead Letter Queues

Write failed records to a separate file for later investigation and reprocessing.

# Python: Write errors to dead letter file
import json

with open('data.jsonl', 'r') as fin, \
     open('processed.jsonl', 'w') as fout, \
     open('errors.jsonl', 'w') as ferr:

    for line_num, line in enumerate(fin, 1):
        try:
            obj = json.loads(line)
            result = process(obj)
            fout.write(json.dumps(result) + '\n')
        except Exception as e:
            error_record = {
                "line_num": line_num,
                "error": str(e),
                "raw_line": line.strip()
            }
            ferr.write(json.dumps(error_record) + '\n')

Fail-Fast for Critical Errors

For critical pipelines, stop immediately on errors to prevent cascading failures.

# Go: Fail fast on parse errors
package main

import (
    "bufio"
    "encoding/json"
    "log"
    "os"
)

func main() {
    file, err := os.Open("data.jsonl")
    if err != nil {
        log.Fatal(err)
    }
    defer file.Close()

    scanner := bufio.NewScanner(file)
    lineNum := 0

    for scanner.Scan() {
        lineNum++
        var record map[string]interface{}

        if err := json.Unmarshal(scanner.Bytes(), &record); err != nil {
            log.Fatalf("Parse error on line %d: %v", lineNum, err)
        }

        // Process record...
    }

    if err := scanner.Err(); err != nil {
        log.Fatal(err)
    }
}

Implement Retry Logic

For transient errors (network, rate limits), implement exponential backoff retry.

# Python: Retry with exponential backoff
import json
import time
import requests

def process_with_retry(obj, max_retries=3):
    for attempt in range(max_retries):
        try:
            response = requests.post('https://api.example.com', json=obj)
            response.raise_for_status()
            return response.json()
        except requests.exceptions.RequestException as e:
            if attempt == max_retries - 1:
                raise
            wait_time = 2 ** attempt  # Exponential backoff: 1s, 2s, 4s
            print(f"Retry {attempt + 1}/{max_retries} after {wait_time}s")
            time.sleep(wait_time)

with open('data.jsonl', 'r') as f:
    for line in f:
        obj = json.loads(line)
        result = process_with_retry(obj)

Checkpointing for Large Files

For multi-GB files, save progress to resume from failures without reprocessing.

# Python: Checkpoint every 10,000 records
import json
import os

CHECKPOINT_FILE = 'progress.txt'
CHECKPOINT_INTERVAL = 10000

# Resume from last checkpoint
start_line = 0
if os.path.exists(CHECKPOINT_FILE):
    with open(CHECKPOINT_FILE, 'r') as f:
        start_line = int(f.read().strip())

with open('huge.jsonl', 'r', encoding='utf-8') as f:
    for line_num, line in enumerate(f):
        # Skip to checkpoint
        if line_num < start_line:
            continue

        obj = json.loads(line)
        process(obj)

        # Save checkpoint periodically
        if (line_num + 1) % CHECKPOINT_INTERVAL == 0:
            with open(CHECKPOINT_FILE, 'w') as cf:
                cf.write(str(line_num + 1))

# Cleanup checkpoint on success
os.remove(CHECKPOINT_FILE)

Use Buffered I/O

Always use buffered readers and writers - a read or write system call per record is what makes naive loops slow. Python's open() is already buffered; the question is how large the buffer is.

with open('data.jsonl', 'r') as f:
    for line in f:  # Default buffering
        obj = json.loads(line)

with open('data.jsonl', 'r', buffering=1024*1024) as f:
    for line in f:
        obj = json.loads(line)

Tip: Python 3.13 and earlier default to an 8 KiB buffer; Python 3.14 raised io.DEFAULT_BUFFER_SIZE to 128 KiB. On older interpreters an explicit 1 MiB buffer is worth measuring on large files; on 3.14 and later the default is usually enough.

Parallelize Processing

Split JSONL files and process chunks in parallel using multiprocessing.

# Python: Parallel processing with multiprocessing
from multiprocessing import Pool
import json

def process_chunk(filename):
    results = []
    with open(filename, 'r') as f:
        for line in f:
            obj = json.loads(line)
            result = expensive_computation(obj)
            results.append(result)
    return results

# Split file into chunks (use split command or custom splitter)
chunk_files = ['chunk-01.jsonl', 'chunk-02.jsonl', 'chunk-03.jsonl', 'chunk-04.jsonl']

# Process in parallel (one process per CPU core). The __main__ guard is
# required: Python 3.14 made forkserver the default start method on Linux,
# and macOS and Windows already spawn, so worker processes re-import this file.
if __name__ == '__main__':
    with Pool() as pool:
        all_results = pool.map(process_chunk, chunk_files)

    # Flatten results
    final_results = [item for sublist in all_results for item in sublist]

Use Fast JSON Libraries

Check the parser before swapping it: some standard libraries have caught up, and some "fast" packages are only fast for one access pattern. Measure on your own records.

# orjson: its README puts loads() at roughly 2x the standard json module
import orjson

with open('data.jsonl', 'rb') as f:
    for line in f:
        obj = orjson.loads(line)  # accepts bytes, no decode step needed

// Node.js: JSON.parse() is the right default per line.
// The simdjson npm binding's own README warns that converting its
// C++ result into JS objects can make parse() slower than JSON.parse();
// its gains come from lazyParse() on large documents, not short JSONL lines.
const obj = JSON.parse(line);

// Go 1.27+: encoding/json now runs on the v2 implementation, and the
// release notes call unmarshal "significantly faster" - no third-party
// parser needed. Code that imports encoding/json gets it on rebuild.
import "encoding/json"

var record MyStruct
json.Unmarshal(line, &record)

Filter Before Parsing

If you only need specific records, use grep or fast text search before JSON parsing.

# Filter with grep first, then parse
grep '"status":"active"' users.jsonl | python process.py

# In Python: Skip parsing if not needed
import json

with open('events.jsonl', 'r') as f:
    for line in f:
        # Quick string check before expensive parsing
        if '"event_type":"click"' not in line:
            continue

        obj = json.loads(line)
        process(obj)

This can provide 10x+ speedup when filtering is selective.

Build Line Offset Indexes

For random access to specific records, build an index mapping record IDs to file offsets.

# Python: Build and use offset index
import json

# Build index (one-time operation)
index = {}  # {record_id: file_offset}
offset = 0

with open('data.jsonl', 'rb') as f:
    while True:
        line = f.readline()
        if not line:
            break

        obj = json.loads(line)
        index[obj['id']] = offset
        offset = f.tell()

# Save index
with open('data.index.json', 'w') as f:
    json.dump(index, f)

# Fast random access using index
with open('data.jsonl', 'rb') as f:
    target_offset = index['record_12345']
    f.seek(target_offset)
    line = f.readline()
    obj = json.loads(line)

Enables O(1) random access instead of O(n) linear scan.