JSONL Performance

Python: JSON vs JSONL Parsing

Test setup: 100,000 records, ~50MB file, Python 3.11, MacBook Pro M1

# Benchmark code
import json
import orjson
import gzip
import time
import tracemalloc

def benchmark_json_array():
    tracemalloc.start()
    start = time.time()
    with open('data.json', 'r') as f:
        data = json.load(f)  # Load entire array
        for record in data:
            process(record)
    elapsed = time.time() - start
    peak = tracemalloc.get_traced_memory()[1] / 1024 / 1024
    tracemalloc.stop()
    print(f"JSON Array: {elapsed*1000:.0f}ms, {peak:.0f}MB")

def benchmark_jsonl_orjson():
    tracemalloc.start()
    start = time.time()
    with open('data.jsonl', 'rb') as f:
        for line in f:
            record = orjson.loads(line)
            process(record)
    elapsed = time.time() - start
    peak = tracemalloc.get_traced_memory()[1] / 1024 / 1024
    tracemalloc.stop()
    print(f"JSONL (orjson): {elapsed*1000:.0f}ms, {peak:.0f}MB")

JavaScript/Node.js: Streaming Performance

Test setup: 100,000 records, ~50MB file, Node.js 20.x, MacBook Pro M1

// Fast JSONL streaming with ndjson
const fs = require('fs');
const ndjson = require('ndjson');

const stream = fs.createReadStream('data.jsonl')
  .pipe(ndjson.parse())
  .on('data', (record) => {
    // Process each record as it arrives
    process(record);
  })
  .on('end', () => {
    console.log('Complete');
  });

Go: Concurrency Benchmark

Test setup: 1,000,000 records, ~500MB file, Go 1.21, MacBook Pro M1, 10 cores

7.6x speedup with parallel processing. JSONL's line-based format is trivially parallelizable.

// Go: Parallel JSONL processing
package main

import (
    "bufio"
    "encoding/json"
    "os"
    "sync"
)

func processChunk(lines [][]byte, wg *sync.WaitGroup) {
    defer wg.Done()
    for _, line := range lines {
        var record map[string]interface{}
        json.Unmarshal(line, &record)
        // Process record...
    }
}

func main() {
    file, _ := os.Open("data.jsonl")
    defer file.Close()

    scanner := bufio.NewScanner(file)
    const chunkSize = 10000
    var chunk [][]byte
    var wg sync.WaitGroup

    for scanner.Scan() {
        chunk = append(chunk, append([]byte(nil), scanner.Bytes()...))

        if len(chunk) >= chunkSize {
            wg.Add(1)
            go processChunk(chunk, &wg)
            chunk = nil
        }
    }

    if len(chunk) > 0 {
        wg.Add(1)
        go processChunk(chunk, &wg)
    }

    wg.Wait()
}

Compression Ratios

JSONL compresses extremely well due to repeated field names. Test file: 100MB JSONL with typical user records.

Streaming Compressed JSONL

Process compressed JSONL without decompressing entire file first:

import gzip
import json

# Stream-decompress and process
with gzip.open('data.jsonl.gz', 'rt') as f:
    for line in f:
        obj = json.loads(line)
        # Memory stays constant!

const fs = require('fs');
const zlib = require('zlib');
const readline = require('readline');

const stream = fs.createReadStream('data.jsonl.gz')
  .pipe(zlib.createGunzip())
  .pipe(readline.createInterface({ input: process.stdin }));

for await (const line of stream) {
  const obj = JSON.parse(line);
  // Process...
}

# Decompress and process on-the-fly with jq
zcat data.jsonl.gz | jq '.name'

# Decompress, filter, compress again
zcat input.jsonl.gz | grep '"status":"active"' | gzip > filtered.jsonl.gz

Performance impact: Decompression adds 40-60% overhead, but file I/O reduction often makes it faster overall, especially with SSDs.