Batch and real-time ASR with scalability built-in

import modal

MODAL_NAME = "nvidia/parakeet-tdt-0.6b-v2"

image = (

    modal.Image.from_registry(f"nvidia/cuda:{tag}", add_python="3.12")

    .pip_install("nemo_toolkit[asr]", "torchaudio", "soundfile",...)

)

@app.cls(gpu="L40S", image=image)

class BatchTranscription():

    @modal.enter()

    def setup(self):

        self.asr_model = nemo.models.ASRModel.from_pretrained(model_name=MODAL_NAME)

    @modal.method()

    async def run_inference(self, audio_filepaths):

        transcriptions = self.asr_model.transcribe(

            audio_filepaths, batch_size=128, num_workers=1

        )

        return transcriptions

transcriber = BatchTranscription()

for result in transcriber.run_inference.map(filepath_batches):

    ...