modal.NetworkFileSystem
class NetworkFileSystem(modal.object.Object)
A shared, writable file system accessible by one or more Modal functions.
By attaching this file system as a mount to one or more functions, they can share and persist data with each other.
Usage
import modal
nfs = modal.NetworkFileSystem.from_name("my-nfs", create_if_missing=True)
app = modal.App()
@app.function(network_file_systems={"/root/foo": nfs})
def f():
pass
@app.function(network_file_systems={"/root/goo": nfs})
def g():
pass
Also see the CLI methods for accessing network file systems:
modal nfs --help
A NetworkFileSystem
can also be useful for some local scripting scenarios, e.g.:
nfs = modal.NetworkFileSystem.lookup("my-network-file-system")
for chunk in nfs.read_file("my_db_dump.csv"):
...
def __init__(self, *args, **kwargs):
new
@staticmethod
def new(cloud: Optional[str] = None):
NetworkFileSystem.new
is deprecated.
Please use NetworkFileSystem.from_name
(for persisted) or NetworkFileSystem.ephemeral
(for ephemeral) network filesystems.
from_name
@staticmethod
def from_name(
label: str,
namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
environment_name: Optional[str] = None,
create_if_missing: bool = False,
) -> "_NetworkFileSystem":
Create a reference to a persisted network filesystem, optionally creating it lazily.
Examples
volume = NetworkFileSystem.from_name("my-volume", create_if_missing=True)
@app.function(network_file_systems={"/vol": volume})
def f():
pass
ephemeral
@classmethod
@contextmanager
def ephemeral(
cls: Type["_NetworkFileSystem"],
client: Optional[_Client] = None,
environment_name: Optional[str] = None,
_heartbeat_sleep: float = EPHEMERAL_OBJECT_HEARTBEAT_SLEEP,
) -> Iterator["_NetworkFileSystem"]:
Creates a new ephemeral network filesystem within a context manager:
Usage:
with NetworkFileSystem.ephemeral() as nfs:
assert nfs.listdir() == []
async with NetworkFileSystem.ephemeral() as nfs:
assert await nfs.listdir() == []
persisted
@staticmethod
def persisted(
label: str,
namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
environment_name: Optional[str] = None,
cloud: Optional[str] = None,
):
Deprecated! Use NetworkFileSystem.from_name(name, create_if_missing=True)
.
persist
def persist(
self,
label: str,
namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
environment_name: Optional[str] = None,
cloud: Optional[str] = None,
):
NetworkFileSystem().persist("my-volume")
is deprecated.
Use NetworkFileSystem.from_name("my-volume", create_if_missing=True)
instead.
lookup
@staticmethod
def lookup(
label: str,
namespace=api_pb2.DEPLOYMENT_NAMESPACE_WORKSPACE,
client: Optional[_Client] = None,
environment_name: Optional[str] = None,
create_if_missing: bool = False,
) -> "_NetworkFileSystem":
Lookup a network file system with a given name
n = modal.NetworkFileSystem.lookup("my-nfs")
print(n.listdir("/"))
write_file
@live_method
def write_file(self, remote_path: str, fp: BinaryIO, progress_cb: Optional[Callable[..., Any]] = None) -> int:
Write from a file object to a path on the network file system, atomically.
Will create any needed parent directories automatically.
If remote_path ends with /
it’s assumed to be a directory and the
file will be uploaded with its current name to that directory.
read_file
@live_method_gen
def read_file(self, path: str) -> Iterator[bytes]:
Read a file from the network file system
iterdir
@live_method_gen
def iterdir(self, path: str) -> Iterator[FileEntry]:
Iterate over all files in a directory in the network file system.
- Passing a directory path lists all files in the directory (names are relative to the directory)
- Passing a file path returns a list containing only that file’s listing description
- Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths)
add_local_file
@live_method
def add_local_file(
self,
local_path: Union[Path, str],
remote_path: Optional[Union[str, PurePosixPath, None]] = None,
progress_cb: Optional[Callable[..., Any]] = None,
):
add_local_dir
@live_method
def add_local_dir(
self,
local_path: Union[Path, str],
remote_path: Optional[Union[str, PurePosixPath, None]] = None,
progress_cb: Optional[Callable[..., Any]] = None,
):
listdir
@live_method
def listdir(self, path: str) -> List[FileEntry]:
List all files in a directory in the network file system.
- Passing a directory path lists all files in the directory (names are relative to the directory)
- Passing a file path returns a list containing only that file’s listing description
- Passing a glob path (including at least one * or ** sequence) returns all files matching that glob path (using absolute paths)
remove_file
@live_method
def remove_file(self, path: str, recursive=False):
Remove a file in a network file system.