Titan

"""
Titan: Large Payload Streaming Parser
Processes multi-gigabyte JSON/NDJSON streams with fixed memory footprint.
"""
from typing import Iterator, Dict, Any
from pydantic import BaseModel

class BatchStats(BaseModel):
    batch_index: int
    records_processed: int
    bytes_streamed: int

class TitanStreamParser:
    def __init__(self, batch_size: int = 1000):
        self.batch_size = batch_size

    def stream_ndjson(self, lines_iterator: Iterator[str]) -> Iterator[BatchStats]:
        current_batch = 0
        total_records = 0
        bytes_counter = 0

        for line in lines_iterator:
            line_str = line.strip()
            if not line_str:
                continue
            bytes_counter += len(line_str)
            total_records += 1
            if total_records % self.batch_size == 0:
                current_batch += 1
                yield BatchStats(
                    batch_index=current_batch,
                    records_processed=total_records,
                    bytes_streamed=bytes_counter
                )