modal.Volume

class Volume(modal.object.Object)

A writeable volume that can be used to share files between one or more Modal functions.

The contents of a volume is exposed as a filesystem. You can use it to share data between different functions, or to persist durable state across several instances of the same function.

Unlike a networked filesystem, you need to explicitly reload the volume to see changes made since it was mounted. Similarly, you need to explicitly commit any changes you make to the volume for the changes to become visible outside the current container.

Concurrent modification is supported, but concurrent modifications of the same files should be avoided! Last write wins in case of concurrent modification of the same file - any data the last writer didn’t have when committing changes will be lost!

As a result, volumes are typically not a good fit for use cases where you need to make concurrent modifications to the same file (nor is distributed file locking supported).

Volumes can only be reloaded if there are no open files for the volume - attempting to reload with open files will result in an error.

Usage

import modal

app = modal.App()  # Note: "app" was called "stub" up until April 2024
volume = modal.Volume.from_name("my-persisted-volume", create_if_missing=True)

@app.function(volumes={"/root/foo": volume})
def f():
    with open("/root/foo/bar.txt", "w") as f:
        f.write("hello")
    volume.commit()  # Persist changes

@app.function(volumes={"/root/foo": volume})
def g():
    volume.reload()  # Fetch latest changes
    with open("/root/foo/bar.txt", "r") as f:
        print(f.read())

def __init__(self, *args, **kwargs):

new

@staticmethod
def new() -> "_Volume":

Volume.new is deprecated.

Please use Volume.from_name (for persisted) or Volume.ephemeral (for ephemeral) volumes.

from_name

@staticmethod
def from_name(
    label: str,
    namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
    environment_name: Optional[str] = None,
    create_if_missing: bool = False,
) -> "_Volume":

Create a reference to a persisted volume. Optionally create it lazily.

Example Usage

import modal

volume = modal.Volume.from_name("my-volume", create_if_missing=True)

app = modal.App()  # Note: "app" was called "stub" up until April 2024

# Volume refers to the same object, even across instances of `app`.
@app.function(volumes={"/vol": volume})
def f():
    pass

ephemeral

@classmethod
@contextmanager
def ephemeral(
    cls: Type["_Volume"],
    client: Optional[_Client] = None,
    environment_name: Optional[str] = None,
    _heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
) -> Iterator["_Volume"]:

Creates a new ephemeral volume within a context manager:

Usage:

with Volume.ephemeral() as vol:
    assert vol.listdir() == []

async with Volume.ephemeral() as vol:
    assert await vol.listdir() == []

persisted

@staticmethod
def persisted(
    label: str,
    namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
    environment_name: Optional[str] = None,
    cloud: Optional[str] = None,
) -> "_Volume":

Deprecated! Use Volume.from_name(name, create_if_missing=True).

lookup

@staticmethod
def lookup(
    label: str,
    namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
    client: Optional[_Client] = None,
    environment_name: Optional[str] = None,
    create_if_missing: bool = False,
) -> "_Volume":

Lookup a volume with a given name

n = modal.Volume.lookup("my-volume")
print(n.listdir("/"))

commit

@live_method
def commit(self):

Commit changes to the volume.

If successful, the changes made are now persisted in durable storage and available to other containers accessing the volume.

reload

@live_method
def reload(self):

Make latest committed state of volume available in the running container.

Any uncommitted changes to the volume, such as new or modified files, may implicitly be committed when reloading.

Reloading will fail if there are open files for the volume.

iterdir

@live_method_gen
def iterdir(self, path: str, *, recursive: bool = True) -> Iterator[FileEntry]:

Iterate over all files in a directory in the volume.

Passing a directory path lists all files in the directory. For a file path, return only that file’s description. If recursive is set to True, list all files and folders under the path recursively.

listdir

@live_method
def listdir(self, path: str, *, recursive: bool = False) -> List[FileEntry]:

List all files under a path prefix in the modal.Volume.

read_file

@live_method_gen
def read_file(self, path: Union[str, bytes]) -> Iterator[bytes]:

Read a file from the modal.Volume.

Example:

vol = modal.Volume.lookup("my-modal-volume")
data = b""
for chunk in vol.read_file("1mb.csv"):
    data += chunk
print(len(data))  # == 1024 * 1024

remove_file

@live_method
def remove_file(self, path: Union[str, bytes], recursive: bool = False) -> None:

Remove a file or directory from a volume.

copy_files

@live_method
def copy_files(self, src_paths: Sequence[Union[str, bytes]], dst_path: Union[str, bytes]) -> None:

Copy files within the volume from src_paths to dst_path. The semantics of the copy operation follow those of the UNIX cp command.

The src_paths parameter is a list. If you want to copy a single file, you should pass a list with a single element.

src_paths and dst_path should refer to the desired location inside the volume. You do not need to prepend the volume mount path.

Usage

vol = modal.Volume.lookup("my-modal-volume")

vol.copy_files(["bar/example.txt"], "bar2")  # Copy files to another directory
vol.copy_files(["bar/example.txt"], "bar/example2.txt")  # Rename a file by copying

Note that if the volume is already mounted on the Modal function, you should use normal filesystem operations like os.rename() and then commit() the volume. The copy_files() method is useful when you don’t have the volume mounted as a filesystem, e.g. when running a script on your local computer.

batch_upload

@live_method
def batch_upload(self, force: bool = False) -> "_VolumeUploadContextManager":

Initiate a batched upload to a volume.

To allow overwriting existing files, set force to True (you cannot overwrite existing directories with uploaded files regardless).

Example:

vol = modal.Volume.lookup("my-modal-volume")

with vol.batch_upload() as batch:
    batch.put_file("local-path.txt", "/remote-path.txt")
    batch.put_directory("/local/directory/", "/remote/directory")
    batch.put_file(io.BytesIO(b"some data"), "/foobar")

delete

def delete(*args, label: str = "", client: Optional[_Client] = None, environment_name: Optional[str] = None):
    # -- Backwards-compatibility section
    # TODO(michael) Upon enforcement of this deprecation, remove *args and the default argument for label=.