Module rsyscall.handle

Classes which own resources and provide the main syscall interfaces

We have several resource-owning classes in this module: FileDescriptor, Pointer, Process, MemoryMapping, etc.

In the analogy to near and far pointers, they are like a near pointer plus a segment register. A more useful analogy is to "handles" from classic Mac OS/PalmOS/16-bit Windows memory management. Like handles, these classes are locked on use with the "borrow" context manager, and they are weakly "relocatable", in that they continue to be valid as the task's segment ids (namespaces) change. See: https://en.wikipedia.org/wiki/Mac_OS_memory_management

However, unlike the MacOS handles that are the origin of the name of this module, these resource-owning classes are garbage collected. Garbage collection should be relied on and preferred over context managers or explicit closing, which are both far too inflexible for large scale resource management.

"""Classes which own resources and provide the main syscall interfaces

We have several resource-owning classes in this module: FileDescriptor, Pointer, Process,
MemoryMapping, etc.

In the analogy to near and far pointers, they are like a near pointer plus a segment
register. A more useful analogy is to "handles" from classic Mac OS/PalmOS/16-bit Windows
memory management. Like handles, these classes are locked on use with the "borrow" context
manager, and they are weakly "relocatable", in that they continue to be valid as the
task's segment ids (namespaces) change. See:
https://en.wikipedia.org/wiki/Mac_OS_memory_management

However, unlike the MacOS handles that are the origin of the name of this module, these
resource-owning classes are garbage collected. Garbage collection should be relied on and
preferred over context managers or explicit closing, which are both far too inflexible for
large scale resource management.

"""
from __future__ import annotations
from rsyscall._raw import ffi, lib # type: ignore
from dataclasses import dataclass
import rsyscall.far
import rsyscall.near
from rsyscall.far import File
import os
import typing as t
import logging
import contextlib
from rsyscall.command import Command
from rsyscall.handle.fd import FileDescriptorTask, BaseFileDescriptor, FDTable
from rsyscall.handle.pointer import Pointer, WrittenPointer, ReadablePointer, LinearPointer
from rsyscall.handle.process import Process, ChildProcess, ThreadProcess, ProcessTask
from rsyscall.near.sysif import UnusableSyscallInterface
logger = logging.getLogger(__name__)

from rsyscall.sched import CLONE, Stack, _unshare
from rsyscall.signal import Siginfo
from rsyscall.fcntl import AT, F, FD
from rsyscall.path import Path
from rsyscall.unistd import SEEK, ArgList, Pipe, OK
from rsyscall.unistd.exec import _execve, _execveat, _exit
from rsyscall.linux.futex import RobustListHead, FutexNode
from rsyscall.sys.wait import W

from rsyscall.fcntl        import               FcntlFileDescriptor
from rsyscall.sys.eventfd  import EventfdTask,  EventFileDescriptor
from rsyscall.sys.timerfd  import TimerfdTask,  TimerFileDescriptor
from rsyscall.sys.epoll    import EpollTask,    EpollFileDescriptor
from rsyscall.sys.inotify  import InotifyTask,  InotifyFileDescriptor
from rsyscall.sys.signalfd import SignalfdTask, SignalFileDescriptor
from rsyscall.sys.mman     import MemoryMappingTask, MappableFileDescriptor
from rsyscall.sys.stat     import               StatFileDescriptor
from rsyscall.signal       import SignalTask
from rsyscall.sys.socket   import SocketTask,   SocketFileDescriptor
from rsyscall.sys.ioctl    import               IoctlFileDescriptor
from rsyscall.linux.dirent import               GetdentsFileDescriptor
from rsyscall.linux.futex  import FutexTask
from rsyscall.linux.memfd  import MemfdTask
from rsyscall.sys.uio      import               UioFileDescriptor
from rsyscall.unistd       import FSTask,       FSFileDescriptor
from rsyscall.unistd.pipe  import PipeTask
from rsyscall.unistd.cwd   import CWDTask
from rsyscall.unistd.credentials import CredentialsTask
from rsyscall.unistd.io    import IOFileDescriptor, SeekableFileDescriptor
from rsyscall.sys.capability import CapabilityTask
from rsyscall.sys.prctl    import PrctlTask
from rsyscall.sys.mount    import MountTask
from rsyscall.sys.resource import ResourceTask
from rsyscall.sched        import SchedTask

# re-exported
from rsyscall.sched import Borrowable

__all__ = [
    "FileDescriptor", "FDTable", "BaseFileDescriptor",
    "Pointer", "WrittenPointer", "ReadablePointer", "LinearPointer",
    "Process", "ChildProcess", "ThreadProcess",
    "Task",
]


################################################################################
# FileDescriptor
T = t.TypeVar('T')
@dataclass(eq=False)
class FileDescriptor(
        EventFileDescriptor, TimerFileDescriptor, EpollFileDescriptor,
        InotifyFileDescriptor, SignalFileDescriptor,
        IoctlFileDescriptor, GetdentsFileDescriptor, UioFileDescriptor,
        SeekableFileDescriptor, IOFileDescriptor,
        FSFileDescriptor,
        SocketFileDescriptor,
        MappableFileDescriptor,
        StatFileDescriptor,
        FcntlFileDescriptor,
        BaseFileDescriptor,
):
    """A file descriptor accessed through some `Task`, with FD-based syscalls as methods

    A `FileDescriptor` represents the ability to use some open file through some `Task`.
    When an open file is created by a syscall in some `Task`,
    the syscall will return a `FileDescriptor` which allows accessing that open file through that `Task`.

    A `FileDescriptor` has many methods to make syscalls;
    most syscalls which take a file descriptor as their first argument are present as a method on `FileDescriptor`.
    These syscalls will be made through the `Task` in the FileDescriptor's `task` field.

    Since there are so many syscalls,
    this class is built by inheriting from many other purpose specific `FooFileDescriptor` classes,
    which in turn all inherit from `BaseFileDescriptor`.

    After we have opened the file and performed some operations on it,
    we can call `close` to immediately close the FileDescriptor and free its resources.
    The FileDescriptor will also be automatically closed in the background
    after the FileDescriptor has been garbage collected.
    Garbage collection should be relied on and preferred over context managers or explicit closing,
    which are both too inflexible for large scale resource management.
    Garbage collection is currently run when we change file descriptor tables,
    as well as on-demand if the user calls `FileDescriptorTask.run_fd_table_gc`.

    We can use `inherit` to copy a FileDescriptor into a task which inherited file descriptors from a parent,
    and `for_task` to copy a FileDescriptor into tasks sharing the same file descriptor table.
    We can also use more complicated methods, such as `rsyscall.sys.socket.CmsgSCMRights`,
    to copy file descriptors without inheritance or a shared file descriptor table.

    """
    __slots__ = ()
    task: Task

    def as_proc_path(self) -> Path:
        """Return the /proc/{pid}/fd/{num} path pointing to this FD.

        This should be used with care, but it's sometimes useful for programs
        which accept paths instead of file descriptors.

        """
        pid = self.task.process.near.id
        num = self.near.number
        return Path(f"/proc/{pid}/fd/{num}")

    async def disable_cloexec(self) -> None:
        "Unset the `FD.CLOEXEC` flag so this file descriptor can be inherited"
        # TODO this doesn't make any sense. we shouldn't allow cloexec if there are multiple people in our fd table;
        # whether or not there are multiple handles to the fd is irrelevant.
        if not self.is_only_handle():
            raise Exception("shouldn't disable cloexec when there are multiple handles to this fd")
        await self.fcntl(F.SETFD, 0)

    async def enable_cloexec(self) -> None:
        "Set the `FD.CLOEXEC` flag so this file descriptor can't be inherited"
        await self.fcntl(F.SETFD, FD.CLOEXEC)

    async def as_argument(self) -> int:
        "`disable_cloexec`, then return this `FileDescriptor` as an integer; useful when passing the FD as an argument"
        await self.disable_cloexec()
        return int(self)

    async def __aenter__(self) -> FileDescriptor:
        return self

    async def __aexit__(self, *args, **kwargs) -> None:
        await self.close()

    def __str__(self) -> str:
        return repr(self)

    def __repr__(self) -> str:
        if self.valid:
            return f"FD({self.task}, {self.near.number})"
        else:
            return f"FD({self.task}, {self.near.number}, valid=False)"


################################################################################
# Task

class Task(
        EventfdTask[FileDescriptor], TimerfdTask[FileDescriptor], EpollTask[FileDescriptor],
        InotifyTask[FileDescriptor], SignalfdTask[FileDescriptor],
        MemfdTask[FileDescriptor],
        FSTask[FileDescriptor],
        SocketTask[FileDescriptor],
        PipeTask,
        MemoryMappingTask, CWDTask,
        FileDescriptorTask[FileDescriptor],
        CapabilityTask, PrctlTask, MountTask,
        CredentialsTask,
        ProcessTask,
        SchedTask,
        ResourceTask,
        FutexTask,
        SignalTask, rsyscall.far.Task,
):
    """A Linux process context under our control, ready for syscalls

    Since there are many different syscalls we could make,
    this class is built by inheriting from many other purpose specific "Task" classes,
    which in turn all inherit from the base `rsyscall.far.Task`.

    This is named after the kernel struct, "struct task", associated with each process.

    """
    def __init__(self,
                 process: t.Union[rsyscall.near.Process, Process],
                 fd_table: FDTable,
                 address_space: rsyscall.far.AddressSpace,
                 pidns: rsyscall.far.PidNamespace,
    ) -> None:
        super().__init__(
            UnusableSyscallInterface(),
            t.cast(rsyscall.near.Process, process), fd_table, address_space, pidns,
        )

    def _file_descriptor_constructor(self, fd: rsyscall.near.FileDescriptor) -> FileDescriptor:
        # for extensibility
        return FileDescriptor(self, fd, True)

    def _make_fresh_address_space(self) -> None:
        self.address_space = rsyscall.far.AddressSpace(self.process.near.id)

    async def unshare(self, flags: CLONE) -> None:
        if flags & CLONE.FILES:
            await self.unshare_files()
            flags ^= CLONE.FILES
        if flags:
            await _unshare(self.sysif, flags)

    async def setns_user(self, fd: FileDescriptor) -> None:
        # can't setns to a user namespace while sharing CLONE_FS
        await self.unshare(CLONE.FS)
        await self.setns(fd, CLONE.NEWUSER)

    async def execveat(self, fd: t.Optional[FileDescriptor],
                       pathname: WrittenPointer[t.Union[str, os.PathLike]],
                       argv: WrittenPointer[ArgList],
                       envp: WrittenPointer[ArgList],
                       flags: AT=AT.NONE,
                       command: Command=None,
    ) -> None:
        with contextlib.ExitStack() as stack:
            if fd:
                fd_n: t.Optional[rsyscall.near.FileDescriptor] = stack.enter_context(fd.borrow(self))
            else:
                fd_n = None
            stack.enter_context(pathname.borrow(self))
            argv.check_address_space(self)
            envp.check_address_space(self)
            for arg in [*argv.value, *envp.value]:
                stack.enter_context(arg.borrow(self))
            self.manipulating_fd_table = True
            try:
                await _execveat(self.sysif, fd_n, pathname.near, argv.near, envp.near, flags)
            except OSError as exn:
                exn.filename = (fd, pathname.value)
                raise
            finally:
                self.manipulating_fd_table = False
            self._make_fresh_fd_table()
            self._make_fresh_address_space()
            if isinstance(self.process, ChildProcess):
                self.process.did_exec(command)
        await self.sysif.close_interface()

    async def execve(self, filename: WrittenPointer[t.Union[str, os.PathLike]],
                     argv: WrittenPointer[ArgList],
                     envp: WrittenPointer[ArgList],
                     command: Command=None,
    ) -> None:
        filename.check_address_space(self)
        argv.check_address_space(self)
        envp.check_address_space(self)
        for arg in [*argv.value, *envp.value]:
            arg.check_address_space(self)
        self.manipulating_fd_table = True
        try:
            await _execve(self.sysif, filename.near, argv.near, envp.near)
        except OSError as exn:
            exn.filename = filename.value
            raise
        self.manipulating_fd_table = False
        self._make_fresh_fd_table()
        self._make_fresh_address_space()
        if isinstance(self.process, ChildProcess):
            self.process.did_exec(command)
        await self.sysif.close_interface()

    async def exit(self, status: int) -> None:
        self.manipulating_fd_table = True
        await _exit(self.sysif, status)
        self.manipulating_fd_table = False
        self._make_fresh_fd_table()
        # close the syscall interface; we don't have to do this since it'll be
        # GC'd, but maybe we want to be tidy in advance.
        await self.sysif.close_interface()

Sub-modules

rsyscall.handle.fd

Fundamental FD ownership and lifecycle …

rsyscall.handle.pointer

rsyscall.handle.process

Classes

class FileDescriptor (task: Task, near: FileDescriptor, valid: bool)

A file descriptor accessed through some Task, with FD-based syscalls as methods

A FileDescriptor represents the ability to use some open file through some Task. When an open file is created by a syscall in some Task, the syscall will return a FileDescriptor which allows accessing that open file through that Task.

A FileDescriptor has many methods to make syscalls; most syscalls which take a file descriptor as their first argument are present as a method on FileDescriptor. These syscalls will be made through the Task in the FileDescriptor's task field.

Since there are so many syscalls, this class is built by inheriting from many other purpose specific FooFileDescriptor classes, which in turn all inherit from BaseFileDescriptor.

After we have opened the file and performed some operations on it, we can call close to immediately close the FileDescriptor and free its resources. The FileDescriptor will also be automatically closed in the background after the FileDescriptor has been garbage collected. Garbage collection should be relied on and preferred over context managers or explicit closing, which are both too inflexible for large scale resource management. Garbage collection is currently run when we change file descriptor tables, as well as on-demand if the user calls FileDescriptorTask.run_fd_table_gc.

We can use inherit to copy a FileDescriptor into a task which inherited file descriptors from a parent, and for_task to copy a FileDescriptor into tasks sharing the same file descriptor table. We can also use more complicated methods, such as CmsgSCMRights, to copy file descriptors without inheritance or a shared file descriptor table.

Expand source code Browse git

@dataclass(eq=False)
class FileDescriptor(
        EventFileDescriptor, TimerFileDescriptor, EpollFileDescriptor,
        InotifyFileDescriptor, SignalFileDescriptor,
        IoctlFileDescriptor, GetdentsFileDescriptor, UioFileDescriptor,
        SeekableFileDescriptor, IOFileDescriptor,
        FSFileDescriptor,
        SocketFileDescriptor,
        MappableFileDescriptor,
        StatFileDescriptor,
        FcntlFileDescriptor,
        BaseFileDescriptor,
):
    """A file descriptor accessed through some `Task`, with FD-based syscalls as methods

    A `FileDescriptor` represents the ability to use some open file through some `Task`.
    When an open file is created by a syscall in some `Task`,
    the syscall will return a `FileDescriptor` which allows accessing that open file through that `Task`.

    A `FileDescriptor` has many methods to make syscalls;
    most syscalls which take a file descriptor as their first argument are present as a method on `FileDescriptor`.
    These syscalls will be made through the `Task` in the FileDescriptor's `task` field.

    Since there are so many syscalls,
    this class is built by inheriting from many other purpose specific `FooFileDescriptor` classes,
    which in turn all inherit from `BaseFileDescriptor`.

    After we have opened the file and performed some operations on it,
    we can call `close` to immediately close the FileDescriptor and free its resources.
    The FileDescriptor will also be automatically closed in the background
    after the FileDescriptor has been garbage collected.
    Garbage collection should be relied on and preferred over context managers or explicit closing,
    which are both too inflexible for large scale resource management.
    Garbage collection is currently run when we change file descriptor tables,
    as well as on-demand if the user calls `FileDescriptorTask.run_fd_table_gc`.

    We can use `inherit` to copy a FileDescriptor into a task which inherited file descriptors from a parent,
    and `for_task` to copy a FileDescriptor into tasks sharing the same file descriptor table.
    We can also use more complicated methods, such as `rsyscall.sys.socket.CmsgSCMRights`,
    to copy file descriptors without inheritance or a shared file descriptor table.

    """
    __slots__ = ()
    task: Task

    def as_proc_path(self) -> Path:
        """Return the /proc/{pid}/fd/{num} path pointing to this FD.

        This should be used with care, but it's sometimes useful for programs
        which accept paths instead of file descriptors.

        """
        pid = self.task.process.near.id
        num = self.near.number
        return Path(f"/proc/{pid}/fd/{num}")

    async def disable_cloexec(self) -> None:
        "Unset the `FD.CLOEXEC` flag so this file descriptor can be inherited"
        # TODO this doesn't make any sense. we shouldn't allow cloexec if there are multiple people in our fd table;
        # whether or not there are multiple handles to the fd is irrelevant.
        if not self.is_only_handle():
            raise Exception("shouldn't disable cloexec when there are multiple handles to this fd")
        await self.fcntl(F.SETFD, 0)

    async def enable_cloexec(self) -> None:
        "Set the `FD.CLOEXEC` flag so this file descriptor can't be inherited"
        await self.fcntl(F.SETFD, FD.CLOEXEC)

    async def as_argument(self) -> int:
        "`disable_cloexec`, then return this `FileDescriptor` as an integer; useful when passing the FD as an argument"
        await self.disable_cloexec()
        return int(self)

    async def __aenter__(self) -> FileDescriptor:
        return self

    async def __aexit__(self, *args, **kwargs) -> None:
        await self.close()

    def __str__(self) -> str:
        return repr(self)

    def __repr__(self) -> str:
        if self.valid:
            return f"FD({self.task}, {self.near.number})"
        else:
            return f"FD({self.task}, {self.near.number}, valid=False)"

Ancestors

• EventFileDescriptor
• TimerFileDescriptor
• EpollFileDescriptor
• InotifyFileDescriptor
• SignalFileDescriptor
• IoctlFileDescriptor
• GetdentsFileDescriptor
• UioFileDescriptor
• SeekableFileDescriptor
• IOFileDescriptor
• FSFileDescriptor
• SocketFileDescriptor
• MappableFileDescriptor
• StatFileDescriptor
• FcntlFileDescriptor
• BaseFileDescriptor

Methods

def as_proc_path(self) ‑> Path

Return the /proc/{pid}/fd/{num} path pointing to this FD.

This should be used with care, but it's sometimes useful for programs which accept paths instead of file descriptors.

Expand source code Browse git

def as_proc_path(self) -> Path:
    """Return the /proc/{pid}/fd/{num} path pointing to this FD.

    This should be used with care, but it's sometimes useful for programs
    which accept paths instead of file descriptors.

    """
    pid = self.task.process.near.id
    num = self.near.number
    return Path(f"/proc/{pid}/fd/{num}")

async def disable_cloexec(self) ‑> NoneType

Unset the FD.CLOEXEC flag so this file descriptor can be inherited

Expand source code Browse git

async def disable_cloexec(self) -> None:
    "Unset the `FD.CLOEXEC` flag so this file descriptor can be inherited"
    # TODO this doesn't make any sense. we shouldn't allow cloexec if there are multiple people in our fd table;
    # whether or not there are multiple handles to the fd is irrelevant.
    if not self.is_only_handle():
        raise Exception("shouldn't disable cloexec when there are multiple handles to this fd")
    await self.fcntl(F.SETFD, 0)

async def enable_cloexec(self) ‑> NoneType

Set the FD.CLOEXEC flag so this file descriptor can't be inherited

Expand source code Browse git

async def enable_cloexec(self) -> None:
    "Set the `FD.CLOEXEC` flag so this file descriptor can't be inherited"
    await self.fcntl(F.SETFD, FD.CLOEXEC)

async def as_argument(self) ‑> int

disable_cloexec, then return this FileDescriptor as an integer; useful when passing the FD as an argument

Expand source code Browse git

async def as_argument(self) -> int:
    "`disable_cloexec`, then return this `FileDescriptor` as an integer; useful when passing the FD as an argument"
    await self.disable_cloexec()
    return int(self)

Inherited members

• EventFileDescriptor:
  • borrow
  • close
  • for_task
  • inherit
  • invalidate
  • move
  • near
  • task
  • valid
• SeekableFileDescriptor:
  • read
  • write
• FSFileDescriptor:
  • dup2
  • faccessat
  • fchmod
  • ftruncate
  • linkat
  • mkdirat
  • openat
  • readlinkat
  • renameat
  • rmdirat
  • symlinkat
  • unlinkat
• SocketFileDescriptor:
  • bind
  • recv
  • recvmsg
  • send
  • sendmsg

class FDTable (creator_pid: int, parent: FDTable = None)

An opaque representation of an existing file descriptor table, compared with "is".

This is the namespace in which a near.FileDescriptor is valid.

For debugging purposes, we take creator_pid as an argument. But pids don't uniquely identify fd tables, because processes can change fd table, such as by calling unshare(CLONE.FILES), while leaving other processes behind in their old address space.

There aren't any useful, efficient identifiers for file descriptor tables, so we compare this object with Python object identify ("is") to see whether two file descriptor tables referenced by two Tasks are the same.

Expand source code Browse git

class FDTable(rsyscall.far.FDTable):
    def __init__(self, creator_pid: int, parent: FDTable=None) -> None:
        super().__init__(creator_pid)
        self.near_to_handles: t.Dict[rsyscall.near.FileDescriptor, WeakSet[BaseFileDescriptor]] = {}
        self.tasks: WeakSet[FileDescriptorTask] = WeakSet([])
        if parent:
            self.inherited: WeakSet[BaseFileDescriptor] = WeakSet(
                itertools.chain(itertools.chain.from_iterable(parent.near_to_handles.values()),
                                parent.inherited)
            )
        else:
            self.inherited = WeakSet()

    def remove_inherited(self) -> None:
        self.inherited = WeakSet()

    def _get_task_in_table(self) -> t.Optional[FileDescriptorTask]:
        for task in list(self.tasks):
            if task.fd_table is not self:
                self.tasks.remove(task)
            elif task.manipulating_fd_table:
                # skip tasks currently changing fd table
                pass
            else:
                return task
        return None

    async def _close_fd(self, task: FileDescriptorTask, fd: rsyscall.near.FileDescriptor) -> None:
        try:
            # TODO we don't have to block here, we can just send off the close without waiting for it,
            # because you aren't supposed to retry close on error.
            # well, except when we actually want to give the user the chance to see the error from close.
            await _close(task.sysif, fd)
        except SyscallHangup:
            # closing the fd through this task went wrong
            # TODO we should mark this task as dead and fall back to later tasks in the list if
            # we fail due to a SyscallInterface-level error; that might happen if, say, this is
            # some decrepit task where we closed the syscallinterface but didn't exit the task.
            assert fd not in self.near_to_handles, f"fd {fd} was somehow reopened before it was actually closed"
            # put the fd back, some other task will close it
            self.near_to_handles[fd] = WeakSet()

    async def gc_using_task(self, task: FileDescriptorTask) -> None:
        gc.collect()
        async with trio.open_nursery() as nursery:
            # take a snapshot of near_to_handles so we can mutate it while iterating
            for fd, handles in list(self.near_to_handles.items()):
                if not handles:
                    # we immediately take responsibility for closing this fd, so our close
                    # attempts don't collide with others
                    del self.near_to_handles[fd]
                    logger.debug("gc for %s: starting close fd for %s", self, fd)
                    nursery.start_soon(self._close_fd, task, fd)

    async def run_gc(self) -> None:
        task = self._get_task_in_table()
        if task is not None:
            await self.gc_using_task(task)

Ancestors

• FDTable

Class variables

var creator_pid : int

Methods

def remove_inherited(self) ‑> NoneType

Expand source code Browse git

def remove_inherited(self) -> None:
    self.inherited = WeakSet()

async def gc_using_task(self, task: FileDescriptorTask) ‑> NoneType

Expand source code Browse git

async def gc_using_task(self, task: FileDescriptorTask) -> None:
    gc.collect()
    async with trio.open_nursery() as nursery:
        # take a snapshot of near_to_handles so we can mutate it while iterating
        for fd, handles in list(self.near_to_handles.items()):
            if not handles:
                # we immediately take responsibility for closing this fd, so our close
                # attempts don't collide with others
                del self.near_to_handles[fd]
                logger.debug("gc for %s: starting close fd for %s", self, fd)
                nursery.start_soon(self._close_fd, task, fd)

async def run_gc(self) ‑> NoneType

Expand source code Browse git

async def run_gc(self) -> None:
    task = self._get_task_in_table()
    if task is not None:
        await self.gc_using_task(task)

class BaseFileDescriptor (task: FileDescriptorTask, near: FileDescriptor, valid: bool)

A file descriptor accessed through some Task

This is an rsyscall-internal base class, which other FileDescriptor objects inherit from for core lifecycle methods. See FileDescriptor for more information.

Expand source code Browse git

@dataclass(eq=False)
class BaseFileDescriptor:
    """A file descriptor accessed through some `Task`

    This is an rsyscall-internal base class,
    which other `FileDescriptor` objects inherit from for core lifecycle methods.
    See `FileDescriptor` for more information.

    """
    __slots__ = ('task', 'near', 'valid')
    task: FileDescriptorTask
    near: rsyscall.near.FileDescriptor
    valid: bool

    def _validate(self) -> None:
        if not self.valid:
            raise Exception("handle is no longer valid")

    def _invalidate(self) -> bool:
        """Invalidate this reference to this file descriptor

        Returns true if we removed the last reference, and are now responsible for closing the FD.

        """
        if self.valid:
            self.valid = False
            handles = self._remove_from_tracking()
            return len(handles) == 0
        else:
            return False

    async def invalidate(self) -> bool:
        """Invalidate this reference to this file descriptor, closing it if necessary

        Returns true if we removed the last reference, and closed the FD.

        We'll use the task inside the last file descriptor to be invalidated to actually
        do the close.
        """
        if self._invalidate():
            # we were the last handle for this fd, we should close it
            logger.debug("invalidating %s, no handles remaining, closing", self)
            fd_table = self.task.fd_table
            del fd_table.near_to_handles[self.near]
            await fd_table._close_fd(self.task, self.near)
            return True
        else:
            logger.debug("invalidating %s, some handles remaining", self)
            return False

    async def close(self) -> None:
        """Close this file descriptor if it's the only handle to it; throwing if there's other handles

        manpage: close(2)
        """
        if not self.is_only_handle():
            raise Exception("can't close this fd", self, "there are handles besides this one to it",
                            self._get_global_handles())
        if not self.valid:
            raise Exception("can't close an invalid FD handle")
        closed = await self.invalidate()
        if not closed:
            raise Exception("for some reason, the fd wasn't closed; "
                            "maybe some race condition where there are still handles left around?")

    def inherit(self, task: FileDescriptorTask[T_fd]) -> T_fd:
        """Make another FileDescriptor referencing the same file but using `task`, which inherited this FD, for syscalls

        Whenever we call `clone` (without passing `CLONE.FILES`),
        the file descriptors in the parent process are copied to the child process;
        this is tracked in rsyscall, and we can call `inherit` on any parent `FileDescriptor`
        to get a handle for the copy in the child.

        """
        return task.inherit_fd(self)

    def for_task(self, task: FileDescriptorTask[T_fd]) -> T_fd:
        """Make a new handle for the same FD, but using `task`, in the same FD table, for syscalls

        Two tasks in the same file descriptor table can be created by calling `clone` with `CLONE.FILES`.

        Once we call this method, we'll have multiple handles for a single file descriptor,
        in a single file descriptor table.
        We won't be able to use `close` to close the FD, since that would break other handles;
        we'll need to use `invalidate` instead.
        (Or, we can just rely on garbage collection.)

        If we want to access the file from another task, we may call the for_task method on
        the FileDescriptor, passing the other task from which we want to access the file.
        This will return another FileDescriptor referencing that file.  This will only work if
        the two tasks are in the same file descriptor table; that is typically the case for
        most scenarios and most kinds of threads. .

        """
        self._validate()
        if self.task.fd_table != task.fd_table:
            raise rsyscall.far.FDTableMismatchError(self.task.fd_table, task.fd_table)
        return task._make_fd_handle_from_near(self.near)

    @contextlib.contextmanager
    def borrow(self, task: FileDescriptorTask) -> t.Iterator[rsyscall.near.FileDescriptor]:
        "Validate that this FD can be accessed from this Task, and yield the near.FD to use for syscalls"
        # TODO we should be the only means of getting FD.near
        # TODO we should just set an in_use flag or something
        # oh argh, what about borrow_with, though?
        # hmm that's fine I guess... there's references inside...
        # ok, the thing is, we already can't move fds or pointers around
        # because we have references in memory
        # maybe borrowing should be another, more strong reference?
        # well, the point of this that we won't be freed during a syscall
        if self.task == task:
            yield self.near
        else:
            # note that we can't immediately change this to not use for_task,
            # because we need to get an FD which stays in the same fd table as task,
            # even if the task owning the FD we're borrowing switches fd tables
            borrowed = self.for_task(task)
            try:
                yield borrowed.near
            finally:
                # we can't call invalidate since we can't actually close this fd since that would
                # require more syscalls. we should really make it so that if the user tries to
                # invalidate the fd they passed into a syscall, they get an exception when they call
                # invalidate. but in lieu of that, we'll throw here. this will cause us to drop
                # events from syscalls, which would break a system that wants to handle exceptions
                # and resume, so we should fix this later. TODO
                # hmm actually I think it might be fine to borrow an fd and free its original?
                # that will happen if we borrow an expression... which should be fine...
                # maybe borrow is a bad design.
                # maybe borrow should just mean, you can't invalidate this fd right now.
                # though we do want to also check that it's the right address space...
                if borrowed.valid:
                    borrowed.valid = False
                    if len(borrowed._remove_from_tracking()) == 0:
                        raise Exception("borrowed fd must have been freed from under us, %s", borrowed)

    def move(self, task: FileDescriptorTask[T_fd]) -> T_fd:
        """Return the output of self.for_task(task), and also invalidate `self`.

        This is useful for more precisely expressing intent, if we don't intend to use
        `self` after getting the new FileDescriptor for the other task.

        This is also somewhat optimized relative to just calling self.for_task then
        calling self.invalidate; the latter call will have to be async, but this call
        doesn't have to be async, since we know we won't be invalidating the last handle.

        """
        new = self.for_task(task)
        self.valid = False
        handles = self._remove_from_tracking()
        if len(handles) == 0:
            raise Exception("We just made handle B from handle A, "
                            "so we know there are at least two handles; "
                            "but after removing handle A, there are no handles left. Huh?")
        return new

    def _get_global_handles(self) -> WeakSet[BaseFileDescriptor]:
        return self.task.fd_table.near_to_handles[self.near]

    def is_only_handle(self) -> bool:
        self._validate()
        return len(self._get_global_handles()) == 1

    def _remove_from_tracking(self) -> WeakSet[BaseFileDescriptor]:
        self.task.fd_handles.remove(self)
        handles = self._get_global_handles()
        handles.remove(self)
        return handles

    def __int__(self) -> int:
        return self.near.number

Subclasses

• FcntlFileDescriptor
• FileDescriptor
• GetdentsFileDescriptor
• EpollFileDescriptor
• EventFileDescriptor
• InotifyFileDescriptor
• IoctlFileDescriptor
• MappableFileDescriptor
• SignalFileDescriptor
• SocketFileDescriptor
• StatFileDescriptor
• TimerFileDescriptor
• UioFileDescriptor
• FSFileDescriptor
• IOFileDescriptor

Instance variables

var near : FileDescriptor

Return an attribute of instance, which is of type owner.

var task : FileDescriptorTask

Return an attribute of instance, which is of type owner.

var valid : bool

Return an attribute of instance, which is of type owner.

Methods

async def invalidate(self) ‑> bool

Invalidate this reference to this file descriptor, closing it if necessary

Returns true if we removed the last reference, and closed the FD.

We'll use the task inside the last file descriptor to be invalidated to actually do the close.

Expand source code Browse git

async def invalidate(self) -> bool:
    """Invalidate this reference to this file descriptor, closing it if necessary

    Returns true if we removed the last reference, and closed the FD.

    We'll use the task inside the last file descriptor to be invalidated to actually
    do the close.
    """
    if self._invalidate():
        # we were the last handle for this fd, we should close it
        logger.debug("invalidating %s, no handles remaining, closing", self)
        fd_table = self.task.fd_table
        del fd_table.near_to_handles[self.near]
        await fd_table._close_fd(self.task, self.near)
        return True
    else:
        logger.debug("invalidating %s, some handles remaining", self)
        return False

async def close(self) ‑> NoneType

Close this file descriptor if it's the only handle to it; throwing if there's other handles

manpage: close(2)

Expand source code Browse git

async def close(self) -> None:
    """Close this file descriptor if it's the only handle to it; throwing if there's other handles

    manpage: close(2)
    """
    if not self.is_only_handle():
        raise Exception("can't close this fd", self, "there are handles besides this one to it",
                        self._get_global_handles())
    if not self.valid:
        raise Exception("can't close an invalid FD handle")
    closed = await self.invalidate()
    if not closed:
        raise Exception("for some reason, the fd wasn't closed; "
                        "maybe some race condition where there are still handles left around?")

def inherit(self, task: FileDescriptorTask[T_fd]) ‑> ~T_fd

Make another FileDescriptor referencing the same file but using task, which inherited this FD, for syscalls

Whenever we call clone (without passing CLONE.FILES), the file descriptors in the parent process are copied to the child process; this is tracked in rsyscall, and we can call inherit on any parent FileDescriptor to get a handle for the copy in the child.

Expand source code Browse git

def inherit(self, task: FileDescriptorTask[T_fd]) -> T_fd:
    """Make another FileDescriptor referencing the same file but using `task`, which inherited this FD, for syscalls

    Whenever we call `clone` (without passing `CLONE.FILES`),
    the file descriptors in the parent process are copied to the child process;
    this is tracked in rsyscall, and we can call `inherit` on any parent `FileDescriptor`
    to get a handle for the copy in the child.

    """
    return task.inherit_fd(self)

def for_task(self, task: FileDescriptorTask[T_fd]) ‑> ~T_fd

Make a new handle for the same FD, but using task, in the same FD table, for syscalls

Two tasks in the same file descriptor table can be created by calling clone with CLONE.FILES.

Once we call this method, we'll have multiple handles for a single file descriptor, in a single file descriptor table. We won't be able to use close to close the FD, since that would break other handles; we'll need to use invalidate instead. (Or, we can just rely on garbage collection.)

If we want to access the file from another task, we may call the for_task method on the FileDescriptor, passing the other task from which we want to access the file. This will return another FileDescriptor referencing that file. This will only work if the two tasks are in the same file descriptor table; that is typically the case for most scenarios and most kinds of threads. .

Expand source code Browse git

def for_task(self, task: FileDescriptorTask[T_fd]) -> T_fd:
    """Make a new handle for the same FD, but using `task`, in the same FD table, for syscalls

    Two tasks in the same file descriptor table can be created by calling `clone` with `CLONE.FILES`.

    Once we call this method, we'll have multiple handles for a single file descriptor,
    in a single file descriptor table.
    We won't be able to use `close` to close the FD, since that would break other handles;
    we'll need to use `invalidate` instead.
    (Or, we can just rely on garbage collection.)

    If we want to access the file from another task, we may call the for_task method on
    the FileDescriptor, passing the other task from which we want to access the file.
    This will return another FileDescriptor referencing that file.  This will only work if
    the two tasks are in the same file descriptor table; that is typically the case for
    most scenarios and most kinds of threads. .

    """
    self._validate()
    if self.task.fd_table != task.fd_table:
        raise rsyscall.far.FDTableMismatchError(self.task.fd_table, task.fd_table)
    return task._make_fd_handle_from_near(self.near)

def borrow(self, task: FileDescriptorTask) ‑> Iterator[FileDescriptor]

Validate that this FD can be accessed from this Task, and yield the near.FD to use for syscalls

Expand source code Browse git

@contextlib.contextmanager
def borrow(self, task: FileDescriptorTask) -> t.Iterator[rsyscall.near.FileDescriptor]:
    "Validate that this FD can be accessed from this Task, and yield the near.FD to use for syscalls"
    # TODO we should be the only means of getting FD.near
    # TODO we should just set an in_use flag or something
    # oh argh, what about borrow_with, though?
    # hmm that's fine I guess... there's references inside...
    # ok, the thing is, we already can't move fds or pointers around
    # because we have references in memory
    # maybe borrowing should be another, more strong reference?
    # well, the point of this that we won't be freed during a syscall
    if self.task == task:
        yield self.near
    else:
        # note that we can't immediately change this to not use for_task,
        # because we need to get an FD which stays in the same fd table as task,
        # even if the task owning the FD we're borrowing switches fd tables
        borrowed = self.for_task(task)
        try:
            yield borrowed.near
        finally:
            # we can't call invalidate since we can't actually close this fd since that would
            # require more syscalls. we should really make it so that if the user tries to
            # invalidate the fd they passed into a syscall, they get an exception when they call
            # invalidate. but in lieu of that, we'll throw here. this will cause us to drop
            # events from syscalls, which would break a system that wants to handle exceptions
            # and resume, so we should fix this later. TODO
            # hmm actually I think it might be fine to borrow an fd and free its original?
            # that will happen if we borrow an expression... which should be fine...
            # maybe borrow is a bad design.
            # maybe borrow should just mean, you can't invalidate this fd right now.
            # though we do want to also check that it's the right address space...
            if borrowed.valid:
                borrowed.valid = False
                if len(borrowed._remove_from_tracking()) == 0:
                    raise Exception("borrowed fd must have been freed from under us, %s", borrowed)

def move(self, task: FileDescriptorTask[T_fd]) ‑> ~T_fd

Return the output of self.for_task(task), and also invalidate self.

This is useful for more precisely expressing intent, if we don't intend to use self after getting the new FileDescriptor for the other task.

This is also somewhat optimized relative to just calling self.for_task then calling self.invalidate; the latter call will have to be async, but this call doesn't have to be async, since we know we won't be invalidating the last handle.

Expand source code Browse git

def move(self, task: FileDescriptorTask[T_fd]) -> T_fd:
    """Return the output of self.for_task(task), and also invalidate `self`.

    This is useful for more precisely expressing intent, if we don't intend to use
    `self` after getting the new FileDescriptor for the other task.

    This is also somewhat optimized relative to just calling self.for_task then
    calling self.invalidate; the latter call will have to be async, but this call
    doesn't have to be async, since we know we won't be invalidating the last handle.

    """
    new = self.for_task(task)
    self.valid = False
    handles = self._remove_from_tracking()
    if len(handles) == 0:
        raise Exception("We just made handle B from handle A, "
                        "so we know there are at least two handles; "
                        "but after removing handle A, there are no handles left. Huh?")
    return new

def is_only_handle(self) ‑> bool

Expand source code Browse git

def is_only_handle(self) -> bool:
    self._validate()
    return len(self._get_global_handles()) == 1

class Pointer (mapping: MemoryMapping, transport: MemoryGateway, serializer: Serializer[T], allocation: AllocationInterface, typ: t.Type[T])

An owning handle for some piece of memory.

More precisely, this is an owning handle for an allocation in some memory mapping. We're explicitly representing memory mappings, rather than glossing over them and pretending that the address space is flat and uniform. If we have two mappings for the same file, we can translate this Pointer between them.

As an implication of owning an allocation, we also know the length of that allocation, which is the length of the range of memory that it's valid to operate on through this pointer. We retrieve this through Pointer.size and use it in many places; anywhere we take a Pointer, if there's some question about what size to operate on, we operate on the full size of the pointer. Reducing the amount of memory to operate on can be done through Pointer.split.

We also know the type of the region of memory; that is, how to interpret this region of memory. This is useful at type-checking time to check that we aren't passing pointers to memory of the wrong type. At runtime, the type is reified as a serializer, which allows us to translate a value of the type to and from bytes.

We also hold a transport which will allow us to read and write the memory we own. Combined with the serializer, this allows us to write and read values of the appropriate type to and from memory using the Pointer.write and Pointer.read methods.

Finally, pointers have a "valid" bit which says whether the Pointer can be used. We say that a method "consumes" a pointer if it will invalidate that pointer.

Most of the methods manipulating the pointer are "linear". That is, they consume the pointer object they're called on and return a new pointer object to use. This forces the user to be more careful with tracking the state of the pointer; and also allows us to represent some state changes with by changing the type of the pointer, in particular Pointer.write.

See also the inheriting class WrittenPointer

Expand source code Browse git

@dataclass(eq=False)
class Pointer(t.Generic[T]):
    """An owning handle for some piece of memory.

    More precisely, this is an owning handle for an allocation in some memory mapping.  We're
    explicitly representing memory mappings, rather than glossing over them and pretending that the
    address space is flat and uniform. If we have two mappings for the same file, we can translate
    this Pointer between them.

    As an implication of owning an allocation, we also know the length of that allocation, which is
    the length of the range of memory that it's valid to operate on through this pointer. We
    retrieve this through Pointer.size and use it in many places; anywhere we take a Pointer, if
    there's some question about what size to operate on, we operate on the full size of the
    pointer. Reducing the amount of memory to operate on can be done through Pointer.split.

    We also know the type of the region of memory; that is, how to interpret this region of
    memory. This is useful at type-checking time to check that we aren't passing pointers to memory
    of the wrong type. At runtime, the type is reified as a serializer, which allows us to translate
    a value of the type to and from bytes.

    We also hold a transport which will allow us to read and write the memory we own. Combined with
    the serializer, this allows us to write and read values of the appropriate type to and from
    memory using the Pointer.write and Pointer.read methods.

    Finally, pointers have a "valid" bit which says whether the Pointer can be used. We say that a
    method "consumes" a pointer if it will invalidate that pointer.

    Most of the methods manipulating the pointer are "linear". That is, they consume the pointer
    object they're called on and return a new pointer object to use. This forces the user to be more
    careful with tracking the state of the pointer; and also allows us to represent some state
    changes with by changing the type of the pointer, in particular Pointer.write.

    See also the inheriting class WrittenPointer

    """
    __slots__ = ('mapping', 'transport', 'serializer', 'allocation', 'valid', 'typ')
    mapping: MemoryMapping
    transport: MemoryGateway
    serializer: Serializer[T]
    allocation: AllocationInterface
    typ: t.Type[T]
    valid: bool

    def __init__(self,
                 mapping: MemoryMapping,
                 transport: MemoryGateway,
                 serializer: Serializer[T],
                 allocation: AllocationInterface,
                 typ: t.Type[T],
    ) -> None:
        self.mapping = mapping
        self.transport = transport
        self.serializer = serializer
        self.allocation = allocation
        self.typ = typ
        self.valid = True

    async def write(self, value: T) -> WrittenPointer[T]:
        "Write this value to this pointer, consuming it and returning a new WrittenPointer"
        self._validate()
        value_bytes = self.serializer.to_bytes(value)
        if len(value_bytes) > self.size():
            raise Exception("value_bytes is too long", len(value_bytes),
                            "for this typed pointer of size", self.size())
        await self.transport.write(self, value_bytes)
        return self._wrote(value)

    async def read(self) -> T:
        "Read the value pointed to by this pointer"
        self._validate()
        value = await self.transport.read(self)
        return self.serializer.from_bytes(value)

    def size(self) -> int:
        """Return the size of this pointer's allocation in bytes

        This is mostly used by syscalls and passed to the kernel, so that the kernel knows the size
        of the buffer that it's been passed. To reduce the size of a buffer passed to the kernel,
        use Pointer.split.

        """
        return self.allocation.size()

    def split(self, size: int) -> t.Tuple[Pointer, Pointer]:
        """Invalidate this pointer and split it into two adjacent pointers

        This is primarily used by syscalls that write to one contiguous part of a buffer and leave
        the rest unused.  They split the pointer into a "used" part and an "unused" part, and return
        both parts.

        """
        self._validate()
        # TODO uhhhh if split throws an exception... don't we need to free... or something...
        self.valid = False
        # TODO we should only allow split if we are the only reference to this allocation
        alloc1, alloc2 = self.allocation.split(size)
        first = self._with_alloc(alloc1)
        # TODO should degrade this pointer to raw bytes or something, or maybe no type at all
        second = self._with_alloc(alloc2)
        return first, second

    def merge(self, ptr: Pointer) -> Pointer:
        """Merge two pointers produced by split back into a single pointer

        The two pointers passed in are invalidated.

        This is primarily used by the user to re-assemble a buffer that was split by a syscall.

        """
        self._validate()
        ptr._validate()
        # TODO should assert that these two pointers both serialize the same thing
        # although they could be different types of serializers...
        self.valid = False
        ptr.valid = False
        # TODO we should only allow merge if we are the only reference to this allocation
        alloc = self.allocation.merge(ptr.allocation)
        return self._with_alloc(alloc)

    def __add__(self, right: Pointer[T]) -> Pointer[T]:
        "left + right desugars to left.merge(right)"
        return self.merge(right)

    def __radd__(self, left: t.Optional[Pointer[T]]) -> Pointer[T]:
        """"left += right" desugars to "left = (left + right) if left is not None else right"

        With this, you can initialize a variable to None, then merge pointers into it in a
        loop. This is especially useful when trying to write an entire buffer, or fill an
        entire buffer by reading.

        """
        if left is None:
            return self
        else:
            return left + self

    @property
    def near(self) -> rsyscall.near.Address:
        """Return the raw memory address referred to by this Pointer

        This is mostly used by syscalls and passed to the kernel, so that the kernel knows the start
        of the buffer to read to or write from.

        """
        # TODO hmm should maybe validate that this fits in the bounds of the mapping I guess
        self._validate()
        try:
            return self.mapping.near.as_address() + self.allocation.offset()
        except UseAfterFreeError as e:
            raise UseAfterFreeError(
                "Allocation inside this Pointer", self,
                "is freed, but the pointer is still valid; someone violated some invariants",
            ) from e

    def check_address_space(self, task: rsyscall.far.Task) -> None:
        if task.address_space != self.mapping.task.address_space:
            raise rsyscall.far.AddressSpaceMismatchError(task.address_space, self.mapping.task.address_space)

    @contextlib.contextmanager
    def borrow(self, task: rsyscall.far.Task) -> t.Iterator[rsyscall.near.Address]:
        """Pin the address of this pointer, and yield the pointer's raw memory address

        We validate this pointer, and pin it in memory so that it can't be moved or deleted while
        it's being used.

        This is mostly used by syscalls and passed to the kernel, so that the kernel knows the start
        of the buffer to read to or write from.

        """
        # TODO actual tracking of pointer references is not yet implemented
        # we should have a flag or lock to indicate that this pointer shouldn't be moved or deleted,
        # while it's being borrowed.
        # TODO rename this to pinned
        # TODO make this the only way to get .near
        self._validate()
        self.check_address_space(task)
        yield self.near

    def _validate(self) -> None:
        if not self.valid:
            raise UseAfterFreeError("handle is no longer valid")

    def free(self) -> None:
        """Free this pointer, invalidating it and releasing the underlying allocation.

        It isn't necessary to explicitly call this, because the pointer will be freed on
        GC. But you can call it anyway if, for example, the pointer will be referenced for
        long after it is done being used.

        """
        if self.valid:
            self.valid = False
            self.allocation.free()

    def __del__(self) -> None:
        # This isn't strictly necessary because the allocation will free itself on __del__.
        # But, that will only happen when *all* pointers referring to the allocation are collected;
        # not just the valid one.
        # So, this ensures GC is a bit more prompt.
        # Oh, wait. The real reason we need this is because the Arena stores references to the allocation.
        # TODO We should fix that.
        self.free()

    def split_from_end(self, size: int, alignment: int) -> t.Tuple[Pointer, Pointer]:
        """Split from the end of this pointer, such that the right pointer is aligned to `alignment`

        Used by write_to_end; mostly only useful for preparing stacks.

        """
        extra_to_remove = (int(self.near) + size) % alignment
        return self.split(self.size() - size - extra_to_remove)

    async def write_to_end(self, value: T, alignment: int) -> t.Tuple[Pointer[T], WrittenPointer[T]]:
        """Write a value to the end of the range of this pointer

        Splits the pointer, and returns both parts.  This function is only useful for preparing
        stacks. Would be nice to figure out either a more generic way to prep stacks, or to figure
        out more things that write_to_end could be used for.

        """
        value_bytes = self.serializer.to_bytes(value)
        rest, write_buf = self.split_from_end(len(value_bytes), alignment)
        written = await write_buf.write(value)
        return rest, written

    def __repr__(self) -> str:
        name = type(self).__name__
        typname = self.typ.__name__
        try:
            return f"{name}[{typname}]({self.near}, {self.size()})"
        except UseAfterFreeError:
            return f"{name}[{typname}](valid={self.valid}, {self.mapping}, {self.allocation}, {self.serializer})"

    #### Various ways to create new Pointers by changing one thing about the old pointer. 
    def _with_mapping(self: T_pointer, mapping: MemoryMapping) -> T_pointer:
        if type(self) is not Pointer:
            raise Exception("subclasses of Pointer must override _with_mapping")
        if mapping.file is not self.mapping.file:
            raise Exception("can only move pointer between two mappings of the same file")
        # we don't have a clean model for referring to the same object through multiple mappings.
        # this is a major TODO.
        # at least two ways to achieve it:
        # - have Pointers become multi-mapping super-pointers, which can be valid in multiple address spaces
        # - break our linearity constraint on pointers, allowing multiple pointers for the same allocation;
        #   this is difficult because split() is only easy to implement due to linearity.
        # right here, we just linearly move the pointer to a new mapping
        self._validate()
        self.valid = False
        return type(self)(mapping, self.transport, self.serializer, self.allocation, self.typ)

    def _with_alloc(self, allocation: AllocationInterface) -> Pointer:
        return Pointer(self.mapping, self.transport, self.serializer, allocation, self.typ)

    def _reinterpret(self, serializer: Serializer[U], typ: t.Type[U]) -> Pointer[U]:
        # TODO how can we check to make sure we don't reinterpret in wacky ways?
        # maybe we should only be able to reinterpret in ways that are allowed by the serializer?
        # so maybe it's a method on the Serializer? cast_to(Type)?
        self._validate()
        self.valid = False
        return Pointer(self.mapping, self.transport, serializer, self.allocation, typ)

    def _readable(self) -> ReadablePointer[T]:
        self._validate()
        self.valid = False
        return ReadablePointer(self.mapping, self.transport, self.serializer, self.allocation, self.typ)

    def readable_split(self, size: int) -> t.Tuple[ReadablePointer[T], Pointer]:
        left, right = self.split(size)
        return left._readable(), right

    def _linearize(self) -> LinearPointer[T]:
        self._validate()
        self.valid = False
        return LinearPointer(self.mapping, self.transport, self.serializer, self.allocation, self.typ)

    def unsafe(self) -> ReadablePointer[T]:
        "Get a ReadablePointer from this pointer, even though it might not be initialized"
        return self._readable()

    def _wrote(self, value: T) -> WrittenPointer[T]:
        "Assert we wrote this value to this pointer, and return the appropriate new WrittenPointer"
        self._validate()
        self.valid = False
        return WrittenPointer(self.mapping, self.transport, value, self.serializer, self.allocation, self.typ)

Ancestors

• typing.Generic

Subclasses

• ReadablePointer
• WrittenPointer

Instance variables

var near : Address

Return the raw memory address referred to by this Pointer

This is mostly used by syscalls and passed to the kernel, so that the kernel knows the start of the buffer to read to or write from.

Expand source code Browse git

@property
def near(self) -> rsyscall.near.Address:
    """Return the raw memory address referred to by this Pointer

    This is mostly used by syscalls and passed to the kernel, so that the kernel knows the start
    of the buffer to read to or write from.

    """
    # TODO hmm should maybe validate that this fits in the bounds of the mapping I guess
    self._validate()
    try:
        return self.mapping.near.as_address() + self.allocation.offset()
    except UseAfterFreeError as e:
        raise UseAfterFreeError(
            "Allocation inside this Pointer", self,
            "is freed, but the pointer is still valid; someone violated some invariants",
        ) from e

var allocation : AllocationInterface

Return an attribute of instance, which is of type owner.

var mapping : MemoryMapping

Return an attribute of instance, which is of type owner.

var serializer : Serializer[~T]

Return an attribute of instance, which is of type owner.

var transport : MemoryGateway

Return an attribute of instance, which is of type owner.

var typ : Type[~T]

Return an attribute of instance, which is of type owner.

var valid : bool

Return an attribute of instance, which is of type owner.

Methods

async def write(self, value: T) ‑> WrittenPointer[~T]

Write this value to this pointer, consuming it and returning a new WrittenPointer

Expand source code Browse git

async def write(self, value: T) -> WrittenPointer[T]:
    "Write this value to this pointer, consuming it and returning a new WrittenPointer"
    self._validate()
    value_bytes = self.serializer.to_bytes(value)
    if len(value_bytes) > self.size():
        raise Exception("value_bytes is too long", len(value_bytes),
                        "for this typed pointer of size", self.size())
    await self.transport.write(self, value_bytes)
    return self._wrote(value)

async def read(self) ‑> ~T

Read the value pointed to by this pointer

Expand source code Browse git

async def read(self) -> T:
    "Read the value pointed to by this pointer"
    self._validate()
    value = await self.transport.read(self)
    return self.serializer.from_bytes(value)

def size(self) ‑> int

Return the size of this pointer's allocation in bytes

This is mostly used by syscalls and passed to the kernel, so that the kernel knows the size of the buffer that it's been passed. To reduce the size of a buffer passed to the kernel, use Pointer.split.

Expand source code Browse git

def size(self) -> int:
    """Return the size of this pointer's allocation in bytes

    This is mostly used by syscalls and passed to the kernel, so that the kernel knows the size
    of the buffer that it's been passed. To reduce the size of a buffer passed to the kernel,
    use Pointer.split.

    """
    return self.allocation.size()

def split(self, size: int) ‑> Tuple[Pointer, Pointer]

Invalidate this pointer and split it into two adjacent pointers

This is primarily used by syscalls that write to one contiguous part of a buffer and leave the rest unused. They split the pointer into a "used" part and an "unused" part, and return both parts.

Expand source code Browse git

def split(self, size: int) -> t.Tuple[Pointer, Pointer]:
    """Invalidate this pointer and split it into two adjacent pointers

    This is primarily used by syscalls that write to one contiguous part of a buffer and leave
    the rest unused.  They split the pointer into a "used" part and an "unused" part, and return
    both parts.

    """
    self._validate()
    # TODO uhhhh if split throws an exception... don't we need to free... or something...
    self.valid = False
    # TODO we should only allow split if we are the only reference to this allocation
    alloc1, alloc2 = self.allocation.split(size)
    first = self._with_alloc(alloc1)
    # TODO should degrade this pointer to raw bytes or something, or maybe no type at all
    second = self._with_alloc(alloc2)
    return first, second

def merge(self, ptr: Pointer) ‑> Pointer

Merge two pointers produced by split back into a single pointer

The two pointers passed in are invalidated.

This is primarily used by the user to re-assemble a buffer that was split by a syscall.

Expand source code Browse git

def merge(self, ptr: Pointer) -> Pointer:
    """Merge two pointers produced by split back into a single pointer

    The two pointers passed in are invalidated.

    This is primarily used by the user to re-assemble a buffer that was split by a syscall.

    """
    self._validate()
    ptr._validate()
    # TODO should assert that these two pointers both serialize the same thing
    # although they could be different types of serializers...
    self.valid = False
    ptr.valid = False
    # TODO we should only allow merge if we are the only reference to this allocation
    alloc = self.allocation.merge(ptr.allocation)
    return self._with_alloc(alloc)

def check_address_space(self, task: Task) ‑> NoneType

Expand source code Browse git

def check_address_space(self, task: rsyscall.far.Task) -> None:
    if task.address_space != self.mapping.task.address_space:
        raise rsyscall.far.AddressSpaceMismatchError(task.address_space, self.mapping.task.address_space)

def borrow(self, task: Task) ‑> Iterator[Address]

Pin the address of this pointer, and yield the pointer's raw memory address

We validate this pointer, and pin it in memory so that it can't be moved or deleted while it's being used.

This is mostly used by syscalls and passed to the kernel, so that the kernel knows the start of the buffer to read to or write from.

Expand source code Browse git

@contextlib.contextmanager
def borrow(self, task: rsyscall.far.Task) -> t.Iterator[rsyscall.near.Address]:
    """Pin the address of this pointer, and yield the pointer's raw memory address

    We validate this pointer, and pin it in memory so that it can't be moved or deleted while
    it's being used.

    This is mostly used by syscalls and passed to the kernel, so that the kernel knows the start
    of the buffer to read to or write from.

    """
    # TODO actual tracking of pointer references is not yet implemented
    # we should have a flag or lock to indicate that this pointer shouldn't be moved or deleted,
    # while it's being borrowed.
    # TODO rename this to pinned
    # TODO make this the only way to get .near
    self._validate()
    self.check_address_space(task)
    yield self.near

def free(self) ‑> NoneType

Free this pointer, invalidating it and releasing the underlying allocation.

It isn't necessary to explicitly call this, because the pointer will be freed on GC. But you can call it anyway if, for example, the pointer will be referenced for long after it is done being used.

Expand source code Browse git

def free(self) -> None:
    """Free this pointer, invalidating it and releasing the underlying allocation.

    It isn't necessary to explicitly call this, because the pointer will be freed on
    GC. But you can call it anyway if, for example, the pointer will be referenced for
    long after it is done being used.

    """
    if self.valid:
        self.valid = False
        self.allocation.free()

def split_from_end(self, size: int, alignment: int) ‑> Tuple[Pointer, Pointer]

Split from the end of this pointer, such that the right pointer is aligned to alignment

Used by write_to_end; mostly only useful for preparing stacks.

Expand source code Browse git

def split_from_end(self, size: int, alignment: int) -> t.Tuple[Pointer, Pointer]:
    """Split from the end of this pointer, such that the right pointer is aligned to `alignment`

    Used by write_to_end; mostly only useful for preparing stacks.

    """
    extra_to_remove = (int(self.near) + size) % alignment
    return self.split(self.size() - size - extra_to_remove)

async def write_to_end(self, value: T, alignment: int) ‑> Tuple[Pointer[~T], WrittenPointer[~T]]

Write a value to the end of the range of this pointer

Splits the pointer, and returns both parts. This function is only useful for preparing stacks. Would be nice to figure out either a more generic way to prep stacks, or to figure out more things that write_to_end could be used for.

Expand source code Browse git

async def write_to_end(self, value: T, alignment: int) -> t.Tuple[Pointer[T], WrittenPointer[T]]:
    """Write a value to the end of the range of this pointer

    Splits the pointer, and returns both parts.  This function is only useful for preparing
    stacks. Would be nice to figure out either a more generic way to prep stacks, or to figure
    out more things that write_to_end could be used for.

    """
    value_bytes = self.serializer.to_bytes(value)
    rest, write_buf = self.split_from_end(len(value_bytes), alignment)
    written = await write_buf.write(value)
    return rest, written

def readable_split(self, size: int) ‑> Tuple[ReadablePointer[~T], Pointer]

Expand source code Browse git

def readable_split(self, size: int) -> t.Tuple[ReadablePointer[T], Pointer]:
    left, right = self.split(size)
    return left._readable(), right

def unsafe(self) ‑> ReadablePointer[~T]

Get a ReadablePointer from this pointer, even though it might not be initialized

Expand source code Browse git

def unsafe(self) -> ReadablePointer[T]:
    "Get a ReadablePointer from this pointer, even though it might not be initialized"
    return self._readable()

class WrittenPointer (mapping: MemoryMapping, transport: MemoryGateway, value: T_co, serializer: Serializer[T_co], allocation: AllocationInterface, typ: t.Type[T_co])

A Pointer with some known value written to it

We have all the normal functionality of a Pointer (see that class for more information), but we also know that we've had some value written to us, and we know what that value is, and it's immediately accessible in Python.

We can also view this with an emphasis on the value: This is some known value, that has been written to some memory location. The value and the pointer are equally important in this class, and both are used by most uses of this class.

We use inheritance so that a WrittenPointer gracefully degrades back to a Pointer, and is invalidated whenever a pointer is invalidated. Specifically, we want anything that writes to a pointer to invalidate this pointer. The invalidation lets us know that this value is no longer necessarily written to this pointer.

For example, syscalls that write to pointers will typically call split. A call to WrittenPointer.split will invalidate the WrittenPointer and return regular Pointers; that's desirable because the syscall likely overwrote whatever value was previously written here.

TODO: We should fix syscalls that write to memory but don't call split so that they invalidate the WrittenPointer. That's mostly syscalls using Sockbufs…

Expand source code Browse git

class WrittenPointer(Pointer[T_co]):
    """A Pointer with some known value written to it

    We have all the normal functionality of a Pointer (see that class for more information), but we
    also know that we've had some value written to us, and we know what that value is, and it's
    immediately accessible in Python.

    We can also view this with an emphasis on the value: This is some known value, that has been
    written to some memory location. The value and the pointer are equally important in this class,
    and both are used by most uses of this class.

    We use inheritance so that a WrittenPointer gracefully degrades back to a Pointer, and is
    invalidated whenever a pointer is invalidated. Specifically, we want anything that writes to a
    pointer to invalidate this pointer. The invalidation lets us know that this value is no longer
    necessarily written to this pointer.

    For example, syscalls that write to pointers will typically call split. A call to
    WrittenPointer.split will invalidate the WrittenPointer and return regular Pointers; that's
    desirable because the syscall likely overwrote whatever value was previously written here.

    TODO: We should fix syscalls that write to memory but don't call split so that they invalidate
    the WrittenPointer. That's mostly syscalls using Sockbufs...

    """
    __slots__ = ('value')
    def __init__(self,
                 mapping: MemoryMapping,
                 transport: MemoryGateway,
                 value: T_co,
                 serializer: Serializer[T_co],
                 allocation: AllocationInterface,
                 typ: t.Type[T_co],
    ) -> None:
        super().__init__(mapping, transport, serializer, allocation, typ)
        self.value = value

    def __repr__(self) -> str:
        name = type(self).__name__
        typname = self.typ.__name__
        try:
            return f"{name}[{typname}]({self.near}, {self.value})"
        except UseAfterFreeError:
            return f"{name}[{typname}](valid={self.valid}, {self.mapping}, {self.allocation}, {self.value})"

    def _with_mapping(self, mapping: MemoryMapping) -> WrittenPointer:
        if type(self) is not WrittenPointer:
            raise Exception("subclasses of WrittenPointer must override _with_mapping")
        if mapping.file is not self.mapping.file:
            raise Exception("can only move pointer between two mappings of the same file")
        # see notes in Pointer._with_mapping
        self._validate()
        self.valid = False
        return type(self)(mapping, self.transport, self.value, self.serializer, self.allocation, self.typ)

Ancestors

• Pointer
• typing.Generic

Instance variables

var value

Return an attribute of instance, which is of type owner.

Inherited members

• Pointer:
  • allocation
  • borrow
  • free
  • mapping
  • merge
  • near
  • read
  • serializer
  • size
  • split
  • split_from_end
  • transport
  • typ
  • unsafe
  • valid
  • write
  • write_to_end

class ReadablePointer (mapping: MemoryMapping, transport: MemoryGateway, serializer: Serializer[T], allocation: AllocationInterface, typ: t.Type[T])

A Pointer that is safely readable

This is returned by functions and syscalls which write some (possibly unknown) pure data to an address in memory, which then can be read and deserialized to get a sensical pure data value rather than nonsense.

Immediately after allocation, a Pointer is returned, rather than a ReadablePointer, to indicate that the pointer is uninitialized, and therefore not safely readable.

This is also returned by Pointer.unsafe(), to support system calls where it's not statically known that a passed Pointer is written to and initialized; ioctls, for example. Tt would be better to have a complete description of the Linux interface, so we could get rid of this unsafety.

This is currently only a marker type, but eventually we'll move the read() method here to ReadablePointer from Pointer, so that reading Pointers is actually not allowed. For now, this is just a hint.

Expand source code Browse git

class ReadablePointer(Pointer[T]):
    """A Pointer that is safely readable

    This is returned by functions and syscalls which write some (possibly
    unknown) pure data to an address in memory, which then can be read and
    deserialized to get a sensical pure data value rather than nonsense.

    Immediately after allocation, a Pointer is returned, rather than a
    ReadablePointer, to indicate that the pointer is uninitialized, and
    therefore not safely readable.

    This is also returned by Pointer.unsafe(), to support system calls where
    it's not statically known that a passed Pointer is written to and
    initialized; ioctls, for example.  Tt would be better to have a complete
    description of the Linux interface, so we could get rid of this unsafety.

    This is currently only a marker type, but eventually we'll move the read()
    method here to ReadablePointer from Pointer, so that reading Pointers is
    actually not allowed. For now, this is just a hint.

    """
    __slots__ = ()

    def _with_mapping(self, mapping: MemoryMapping) -> ReadablePointer:
        # see notes in Pointer._with_mapping
        if type(self) is not ReadablePointer:
            raise Exception("subclasses of ReadablePointer must override _with_mapping")
        if mapping.file is not self.mapping.file:
            raise Exception("can only move pointer between two mappings of the same file")
        self._validate()
        self.valid = False
        return type(self)(mapping, self.transport, self.serializer, self.allocation, self.typ)

Ancestors

• Pointer
• typing.Generic

Subclasses

• LinearPointer

Inherited members

• Pointer:
  • allocation
  • borrow
  • free
  • mapping
  • merge
  • near
  • read
  • serializer
  • size
  • split
  • split_from_end
  • transport
  • typ
  • unsafe
  • valid
  • write
  • write_to_end

class LinearPointer (mapping: MemoryMapping, transport: MemoryGateway, serializer: Serializer[T], allocation: AllocationInterface, typ: t.Type[T])

A Pointer that must be read, once

This is returned by functions and syscalls which write a unknown value to an address in memory, which then must be read and deserialized once to manage the resources described by that value, such as file descriptors.

The value is:

• "affine"; it must be read at least once, so that the resources inside can be returned as managed objects.
• "relevant"; it must be read at most once, so that dangling handles to the resources can't be created again after they're closed.

Since it's both affine and relevant, this is a true linear type.

Unfortunately it's going to be quite difficult to guarantee relevance. There are three issues here:

1. The pointer can simply be dropped and garbage collected.
2. System calls can write to the pointer and discard its previous results
3. We can write to the pointer (through Pointer.write()) and discard its previous results

We can mitigate 1 a little by warning in __del__.

We could statically prevent 3 by removing the Pointer.read() and Pointer.write() methods from this class, and only allowing LinearPointer.linear_read(), or dynamically by throwing in write if been_read is false.

Any approach to 2 is going to require some tweaks to the pointer API, and probably some mass changes to syscall implementations. Although maybe we could do it off of .near accesses.

Expand source code Browse git

class LinearPointer(ReadablePointer[T]):
    """A Pointer that must be read, once

    This is returned by functions and syscalls which write a unknown
    value to an address in memory, which then must be read and
    deserialized *once* to manage the resources described by that
    value, such as file descriptors.

    The value is:

    - "affine"; it must be read at least once, so that the resources inside
      can be returned as managed objects.
    - "relevant"; it must be read at most once, so that dangling handles to the
      resources can't be created again after they're closed.

    Since it's both affine and relevant, this is a true linear type.

    Unfortunately it's going to be quite difficult to guarantee relevance. There
    are three issues here:

    1. The pointer can simply be dropped and garbage collected.
    2. System calls can write to the pointer and discard its previous results
    3. We can write to the pointer (through `Pointer.write`) and discard its previous results

    We can mitigate 1 a little by warning in `__del__`.

    We could statically prevent 3 by removing the `Pointer.read` and `Pointer.write` methods
    from this class, and only allowing `LinearPointer.linear_read`, or dynamically by throwing
    in `write` if `been_read` is false.

    Any approach to 2 is going to require some tweaks to the pointer API, and
    probably some mass changes to syscall implementations. Although maybe we
    could do it off of .near accesses.

    """
    __slots__ = ('been_read')

    def __init__(self,
                 mapping: MemoryMapping,
                 transport: MemoryGateway,
                 serializer: Serializer[T],
                 allocation: AllocationInterface,
                 typ: t.Type[T],
    ) -> None:
        super().__init__(mapping, transport, serializer, allocation, typ)
        self.been_read = False

    async def read(self) -> T:
        if self.been_read:
            raise Exception("This LinearPointer has already been read, it can't be read again for safety reasons.")
        ret = await super().read()
        self.been_read = True
        return ret

    async def linear_read(self) -> t.Tuple[T, Pointer[T]]:
        "Read the value, and return the now-inert buffer left over as a Pointer."
        ret = await self.read()
        self.valid = False
        new_ptr = Pointer(self.mapping, self.transport, self.serializer, self.allocation, self.typ)
        return ret, new_ptr

    def __del__(self) -> None:
        super().__del__()
        if not self.been_read:
            logger.error("Didn't read this LinearPointer before dropping it: %s", self)

    def _with_mapping(self, mapping: MemoryMapping) -> LinearPointer:
        # see notes in Pointer._with_mapping
        if type(self) is not LinearPointer:
            raise Exception("subclasses of LinearPointer must override _with_mapping")
        if mapping.file is not self.mapping.file:
            raise Exception("can only move pointer between two mappings of the same file")
        self._validate()
        self.valid = False
        return type(self)(mapping, self.transport, self.serializer, self.allocation, self.typ)

Ancestors

• ReadablePointer
• Pointer
• typing.Generic

Instance variables

var been_read

Return an attribute of instance, which is of type owner.

Methods

async def linear_read(self) ‑> Tuple[~T, Pointer[~T]]

Read the value, and return the now-inert buffer left over as a Pointer.

Expand source code Browse git

async def linear_read(self) -> t.Tuple[T, Pointer[T]]:
    "Read the value, and return the now-inert buffer left over as a Pointer."
    ret = await self.read()
    self.valid = False
    new_ptr = Pointer(self.mapping, self.transport, self.serializer, self.allocation, self.typ)
    return ret, new_ptr

Inherited members

• ReadablePointer:
  • allocation
  • borrow
  • free
  • mapping
  • merge
  • near
  • read
  • serializer
  • size
  • split
  • split_from_end
  • transport
  • typ
  • unsafe
  • valid
  • write
  • write_to_end

class Process (task: Task, near: Process)

A reference to an arbitrary process on the system, not necessarily our child.

This is essentially the equivalent of an arbitrary pid. It's not safe to signal arbitrary pids, because a process exit + pid wrap can cause you to signal some other unexpected process.

It's only safe to signal child processes, using the ChildProcess class. But since it's common to want to signal arbitrary pids even despite the danger, we provide this convenient class to do so.

Expand source code Browse git

@dataclass
class Process:
    """A reference to an arbitrary process on the system, not necessarily our child.

    This is essentially the equivalent of an arbitrary pid. It's not safe to signal
    arbitrary pids, because a process exit + pid wrap can cause you to signal some other
    unexpected process.

    It's only safe to signal child processes, using the ChildProcess class.  But since
    it's common to want to signal arbitrary pids even despite the danger, we provide this
    convenient class to do so.

    """
    task: rsyscall.far.Task
    near: rsyscall.near.Process

    async def kill(self, sig: SIG) -> None:
        await _kill(self.task.sysif, self.near, sig)

    def _as_process_group(self) -> rsyscall.near.ProcessGroup:
        return rsyscall.near.ProcessGroup(self.near.id)

    async def killpg(self, sig: SIG) -> None:
        await _kill(self.task.sysif, self._as_process_group(), sig)

    async def getpgid(self) -> rsyscall.near.ProcessGroup:
        return (await _getpgid(self.task.sysif, self.near))

    async def setpriority(self, prio: int) -> None:
        return (await _setpriority(self.task.sysif, PRIO.PROCESS, self.near.id, prio))

    async def getpriority(self) -> int:
        return (await _getpriority(self.task.sysif, PRIO.PROCESS, self.near.id))

    def __repr__(self) -> str:
        name = type(self).__name__
        return f"{name}({self.near}, parent={self.task})"

Subclasses

• ChildProcess

Class variables

var task : Task

var near : Process

Methods

async def kill(self, sig: SIG) ‑> NoneType

Expand source code Browse git

async def kill(self, sig: SIG) -> None:
    await _kill(self.task.sysif, self.near, sig)

async def killpg(self, sig: SIG) ‑> NoneType

Expand source code Browse git

async def killpg(self, sig: SIG) -> None:
    await _kill(self.task.sysif, self._as_process_group(), sig)

async def getpgid(self) ‑> ProcessGroup

Expand source code Browse git

async def getpgid(self) -> rsyscall.near.ProcessGroup:
    return (await _getpgid(self.task.sysif, self.near))

async def setpriority(self, prio: int) ‑> NoneType

Expand source code Browse git

async def setpriority(self, prio: int) -> None:
    return (await _setpriority(self.task.sysif, PRIO.PROCESS, self.near.id, prio))

async def getpriority(self) ‑> int

Expand source code Browse git

async def getpriority(self) -> int:
    return (await _getpriority(self.task.sysif, PRIO.PROCESS, self.near.id))

class ChildProcess (task: Task, near: Process, alive=True)

A process that is our child, which we can monitor with waitid and safely signal.

Because a child process's pid will not be reused until we wait on its zombie, we can (as long as we're careful about ordering calls to waitid and kill) safely send signals to child processes without the possibility of signaling some other unexpected process.

Expand source code Browse git

class ChildProcess(Process):
    """A process that is our child, which we can monitor with waitid and safely signal.

    Because a child process's pid will not be reused until we wait on its zombie, we can
    (as long as we're careful about ordering calls to waitid and kill) safely send signals
    to child processes without the possibility of signaling some other unexpected process.

    """
    def __init__(self, task: rsyscall.far.Task, near: rsyscall.near.Process, alive=True) -> None:
        self.task = task
        self.near = near
        self.death_state: t.Optional[ChildState] = None
        self.unread_siginfo: t.Optional[Pointer[Siginfo]] = None
        self.in_use = False
        # the command this process exec'd; primarily useful for debugging, but we justify
        # its presence by thinking that the Command might hold references to resources
        # that the process is now using, like temporary directory paths.
        self.command: t.Optional[Command] = None

    def mark_dead(self, state: ChildState) -> None:
        self.death_state = state

    def did_exec(self, command: t.Optional[Command]) -> ChildProcess:
        self.command = command
        return self

    @contextlib.contextmanager
    def borrow(self) -> t.Iterator[None]:
        if self.death_state:
            raise Exception("child process", self.near, "is no longer alive, so we can't wait on it or kill it")
        if self.unread_siginfo:
            raise Exception("for child process", self.near, "waitid or kill was call "
                            "before processing the siginfo buffer from an earlier waitid")
        if self.in_use:
            # TODO technically we could have multiple kills happening simultaneously.
            # but indeed, we can't have a kill happen while a wait is happening, nor multiple waits at a time.
            # that would be racy - we might kill the wrong process or wait on the wrong process
            raise Exception("child process", self.near, "is currently being waited on or killed,"
                            " can't use it a second time")
        self.in_use = True
        try:
            yield
        finally:
            self.in_use = False

    async def kill(self, sig: SIG) -> None:
        with self.borrow():
            await super().kill(sig)

    async def killpg(self, sig: SIG) -> None:
        # This call will throw an error if this child isn't a process group leader, but
        # it's at least guaranteed to not kill some random unrelated process group.
        with self.borrow():
            await super().killpg(sig)

    async def getpgid(self) -> rsyscall.near.ProcessGroup:
        with self.borrow():
            return await super().getpgid()

    async def setpgid(self, pgid: t.Optional[ChildProcess]) -> None:
        # the ownership model of process groups is such that the only way that
        # it's safe to use setpgid on a child process is if we're setpgid-ing to
        # the process group of another child process.
        with self.borrow():
            if pgid is None:
                await _setpgid(self.task.sysif, self.near, None)
            else:
                if pgid.task.pidns != self.task.pidns:
                    raise rsyscall.far.NamespaceMismatchError(
                        "different pid namespaces", pgid.task.pidns, self.task.pidns)
                with pgid.borrow():
                    await _setpgid(self.task.sysif, self.near, self._as_process_group())

    async def waitid(self, options: W, infop: Pointer[Siginfo],
                     *, rusage: t.Optional[Pointer[Siginfo]]=None) -> None:
        with contextlib.ExitStack() as stack:
            stack.enter_context(self.borrow())
            stack.enter_context(infop.borrow(self.task))
            if rusage is not None:
                stack.enter_context(rusage.borrow(self.task))
            try:
                await _waitid(self.task.sysif, self.near, infop.near, options,
                              rusage.near if rusage else None)
            except ChildProcessError as exn:
                exn.filename = self.near
                raise
        self.unread_siginfo = infop

    def parse_waitid_siginfo(self, siginfo: Siginfo) -> t.Optional[ChildState]:
        self.unread_siginfo = None
        # this is to catch the case where we did waitid(W.NOHANG) and there was no event
        if siginfo.pid == 0:
            return None
        else:
            state = ChildState.make_from_siginfo(siginfo)
            if state.died():
                self.mark_dead(state)
            return state

    # helpers
    async def read_siginfo(self) -> t.Optional[ChildState]:
        if self.unread_siginfo is None:
            raise Exception("no siginfo buf to read")
        else:
            siginfo = await self.unread_siginfo.read()
            return self.parse_waitid_siginfo(siginfo)

    async def read_state_change(self) -> ChildState:
        state = await self.read_siginfo()
        if state is None:
            raise Exception("expected a state change, but siginfo buf didn't contain one")
        return state

Ancestors

• Process

Subclasses

• ThreadProcess

Class variables

var task : Task

var near : Process

Methods

def mark_dead(self, state: ChildState) ‑> NoneType

Expand source code Browse git

def mark_dead(self, state: ChildState) -> None:
    self.death_state = state

def did_exec(self, command: t.Optional[Command]) ‑> ChildProcess

Expand source code Browse git

def did_exec(self, command: t.Optional[Command]) -> ChildProcess:
    self.command = command
    return self

def borrow(self) ‑> Iterator[NoneType]

Expand source code Browse git

@contextlib.contextmanager
def borrow(self) -> t.Iterator[None]:
    if self.death_state:
        raise Exception("child process", self.near, "is no longer alive, so we can't wait on it or kill it")
    if self.unread_siginfo:
        raise Exception("for child process", self.near, "waitid or kill was call "
                        "before processing the siginfo buffer from an earlier waitid")
    if self.in_use:
        # TODO technically we could have multiple kills happening simultaneously.
        # but indeed, we can't have a kill happen while a wait is happening, nor multiple waits at a time.
        # that would be racy - we might kill the wrong process or wait on the wrong process
        raise Exception("child process", self.near, "is currently being waited on or killed,"
                        " can't use it a second time")
    self.in_use = True
    try:
        yield
    finally:
        self.in_use = False

async def kill(self, sig: SIG) ‑> NoneType

Expand source code Browse git

async def kill(self, sig: SIG) -> None:
    with self.borrow():
        await super().kill(sig)

async def killpg(self, sig: SIG) ‑> NoneType

Expand source code Browse git

async def killpg(self, sig: SIG) -> None:
    # This call will throw an error if this child isn't a process group leader, but
    # it's at least guaranteed to not kill some random unrelated process group.
    with self.borrow():
        await super().killpg(sig)

async def getpgid(self) ‑> ProcessGroup

Expand source code Browse git

async def getpgid(self) -> rsyscall.near.ProcessGroup:
    with self.borrow():
        return await super().getpgid()

async def setpgid(self, pgid: t.Optional[ChildProcess]) ‑> NoneType

Expand source code Browse git

async def setpgid(self, pgid: t.Optional[ChildProcess]) -> None:
    # the ownership model of process groups is such that the only way that
    # it's safe to use setpgid on a child process is if we're setpgid-ing to
    # the process group of another child process.
    with self.borrow():
        if pgid is None:
            await _setpgid(self.task.sysif, self.near, None)
        else:
            if pgid.task.pidns != self.task.pidns:
                raise rsyscall.far.NamespaceMismatchError(
                    "different pid namespaces", pgid.task.pidns, self.task.pidns)
            with pgid.borrow():
                await _setpgid(self.task.sysif, self.near, self._as_process_group())

async def waitid(self, options: W, infop: Pointer[Siginfo], *, rusage: t.Optional[Pointer[Siginfo]] = None) ‑> NoneType

Expand source code Browse git

async def waitid(self, options: W, infop: Pointer[Siginfo],
                 *, rusage: t.Optional[Pointer[Siginfo]]=None) -> None:
    with contextlib.ExitStack() as stack:
        stack.enter_context(self.borrow())
        stack.enter_context(infop.borrow(self.task))
        if rusage is not None:
            stack.enter_context(rusage.borrow(self.task))
        try:
            await _waitid(self.task.sysif, self.near, infop.near, options,
                          rusage.near if rusage else None)
        except ChildProcessError as exn:
            exn.filename = self.near
            raise
    self.unread_siginfo = infop

def parse_waitid_siginfo(self, siginfo: Siginfo) ‑> Optional[ChildState]

Expand source code Browse git

def parse_waitid_siginfo(self, siginfo: Siginfo) -> t.Optional[ChildState]:
    self.unread_siginfo = None
    # this is to catch the case where we did waitid(W.NOHANG) and there was no event
    if siginfo.pid == 0:
        return None
    else:
        state = ChildState.make_from_siginfo(siginfo)
        if state.died():
            self.mark_dead(state)
        return state

async def read_siginfo(self) ‑> Optional[ChildState]

Expand source code Browse git

async def read_siginfo(self) -> t.Optional[ChildState]:
    if self.unread_siginfo is None:
        raise Exception("no siginfo buf to read")
    else:
        siginfo = await self.unread_siginfo.read()
        return self.parse_waitid_siginfo(siginfo)

async def read_state_change(self) ‑> ChildState

Expand source code Browse git

async def read_state_change(self) -> ChildState:
    state = await self.read_siginfo()
    if state is None:
        raise Exception("expected a state change, but siginfo buf didn't contain one")
    return state

class ThreadProcess (task: Task, near: Process, used_stack: Pointer[Stack], stack_data: Stack, ctid: t.Optional[Pointer[FutexNode]], tls: t.Optional[Pointer])

A child process with some additional stuff, just useful for resource tracking for threads.

We need to free the resources used by our child processes when they die. This class makes that more straightforward.

Expand source code Browse git

class ThreadProcess(ChildProcess):
    """A child process with some additional stuff, just useful for resource tracking for threads.

    We need to free the resources used by our child processes when they die. This class
    makes that more straightforward.

    """
    def __init__(self, task: rsyscall.far.Task, near: rsyscall.near.Process,
                 used_stack: Pointer[Stack],
                 stack_data: Stack,
                 ctid: t.Optional[Pointer[FutexNode]],
                 tls: t.Optional[Pointer],
    ) -> None:
        super().__init__(task, near)
        self.used_stack = used_stack
        self.stack_data = stack_data
        self.ctid = ctid
        self.tls = tls

    def free_everything(self) -> None:
        # TODO don't know how to free the stack data...
        if self.used_stack.valid:
            self.used_stack.free()
        if self.ctid is not None and self.ctid.valid:
            self.ctid.free()
        if self.tls is not None and self.tls.valid:
            self.tls.free()

    def mark_dead(self, event: ChildState) -> None:
        self.free_everything()
        return super().mark_dead(event)

    def did_exec(self, command: t.Optional[Command]) -> ChildProcess:
        self.free_everything()
        return super().did_exec(command)

    def __repr__(self) -> str:
        name = type(self).__name__
        if self.command or self.death_state:
            # pretend to be a ChildProcess if we've exec'd or died
            name = 'ChildProcess'
        return f"{name}({self.near}, parent={self.task})"

Ancestors

• ChildProcess
• Process

Class variables

var task : Task

var near : Process

Methods

def free_everything(self) ‑> NoneType

Expand source code Browse git

def free_everything(self) -> None:
    # TODO don't know how to free the stack data...
    if self.used_stack.valid:
        self.used_stack.free()
    if self.ctid is not None and self.ctid.valid:
        self.ctid.free()
    if self.tls is not None and self.tls.valid:
        self.tls.free()

def mark_dead(self, event: ChildState) ‑> NoneType

Expand source code Browse git

def mark_dead(self, event: ChildState) -> None:
    self.free_everything()
    return super().mark_dead(event)

def did_exec(self, command: t.Optional[Command]) ‑> ChildProcess

Expand source code Browse git

def did_exec(self, command: t.Optional[Command]) -> ChildProcess:
    self.free_everything()
    return super().did_exec(command)

class Task (process: t.Union[Process, Process], fd_table: FDTable, address_space: AddressSpace, pidns: PidNamespace)

A Linux process context under our control, ready for syscalls

Since there are many different syscalls we could make, this class is built by inheriting from many other purpose specific "Task" classes, which in turn all inherit from the base Task.

This is named after the kernel struct, "struct task", associated with each process.

Expand source code Browse git

class Task(
        EventfdTask[FileDescriptor], TimerfdTask[FileDescriptor], EpollTask[FileDescriptor],
        InotifyTask[FileDescriptor], SignalfdTask[FileDescriptor],
        MemfdTask[FileDescriptor],
        FSTask[FileDescriptor],
        SocketTask[FileDescriptor],
        PipeTask,
        MemoryMappingTask, CWDTask,
        FileDescriptorTask[FileDescriptor],
        CapabilityTask, PrctlTask, MountTask,
        CredentialsTask,
        ProcessTask,
        SchedTask,
        ResourceTask,
        FutexTask,
        SignalTask, rsyscall.far.Task,
):
    """A Linux process context under our control, ready for syscalls

    Since there are many different syscalls we could make,
    this class is built by inheriting from many other purpose specific "Task" classes,
    which in turn all inherit from the base `rsyscall.far.Task`.

    This is named after the kernel struct, "struct task", associated with each process.

    """
    def __init__(self,
                 process: t.Union[rsyscall.near.Process, Process],
                 fd_table: FDTable,
                 address_space: rsyscall.far.AddressSpace,
                 pidns: rsyscall.far.PidNamespace,
    ) -> None:
        super().__init__(
            UnusableSyscallInterface(),
            t.cast(rsyscall.near.Process, process), fd_table, address_space, pidns,
        )

    def _file_descriptor_constructor(self, fd: rsyscall.near.FileDescriptor) -> FileDescriptor:
        # for extensibility
        return FileDescriptor(self, fd, True)

    def _make_fresh_address_space(self) -> None:
        self.address_space = rsyscall.far.AddressSpace(self.process.near.id)

    async def unshare(self, flags: CLONE) -> None:
        if flags & CLONE.FILES:
            await self.unshare_files()
            flags ^= CLONE.FILES
        if flags:
            await _unshare(self.sysif, flags)

    async def setns_user(self, fd: FileDescriptor) -> None:
        # can't setns to a user namespace while sharing CLONE_FS
        await self.unshare(CLONE.FS)
        await self.setns(fd, CLONE.NEWUSER)

    async def execveat(self, fd: t.Optional[FileDescriptor],
                       pathname: WrittenPointer[t.Union[str, os.PathLike]],
                       argv: WrittenPointer[ArgList],
                       envp: WrittenPointer[ArgList],
                       flags: AT=AT.NONE,
                       command: Command=None,
    ) -> None:
        with contextlib.ExitStack() as stack:
            if fd:
                fd_n: t.Optional[rsyscall.near.FileDescriptor] = stack.enter_context(fd.borrow(self))
            else:
                fd_n = None
            stack.enter_context(pathname.borrow(self))
            argv.check_address_space(self)
            envp.check_address_space(self)
            for arg in [*argv.value, *envp.value]:
                stack.enter_context(arg.borrow(self))
            self.manipulating_fd_table = True
            try:
                await _execveat(self.sysif, fd_n, pathname.near, argv.near, envp.near, flags)
            except OSError as exn:
                exn.filename = (fd, pathname.value)
                raise
            finally:
                self.manipulating_fd_table = False
            self._make_fresh_fd_table()
            self._make_fresh_address_space()
            if isinstance(self.process, ChildProcess):
                self.process.did_exec(command)
        await self.sysif.close_interface()

    async def execve(self, filename: WrittenPointer[t.Union[str, os.PathLike]],
                     argv: WrittenPointer[ArgList],
                     envp: WrittenPointer[ArgList],
                     command: Command=None,
    ) -> None:
        filename.check_address_space(self)
        argv.check_address_space(self)
        envp.check_address_space(self)
        for arg in [*argv.value, *envp.value]:
            arg.check_address_space(self)
        self.manipulating_fd_table = True
        try:
            await _execve(self.sysif, filename.near, argv.near, envp.near)
        except OSError as exn:
            exn.filename = filename.value
            raise
        self.manipulating_fd_table = False
        self._make_fresh_fd_table()
        self._make_fresh_address_space()
        if isinstance(self.process, ChildProcess):
            self.process.did_exec(command)
        await self.sysif.close_interface()

    async def exit(self, status: int) -> None:
        self.manipulating_fd_table = True
        await _exit(self.sysif, status)
        self.manipulating_fd_table = False
        self._make_fresh_fd_table()
        # close the syscall interface; we don't have to do this since it'll be
        # GC'd, but maybe we want to be tidy in advance.
        await self.sysif.close_interface()

Ancestors

• EventfdTask
• rsyscall.sys.timerfd.TimerfdTask
• EpollTask
• rsyscall.sys.inotify.InotifyTask
• SignalfdTask
• MemfdTask
• rsyscall.unistd.FSTask
• SocketTask
• PipeTask
• MemoryMappingTask
• CWDTask
• FileDescriptorTask
• rsyscall.sys.capability.CapabilityTask
• PrctlTask
• MountTask
• CredentialsTask
• ProcessTask
• rsyscall.sched.SchedTask
• ResourceTask
• FutexTask
• SignalTask
• Task
• typing.Generic

Class variables

var sysif : SyscallInterface

var near_process : Process

var fd_table : FDTable

var address_space : AddressSpace

var pidns : PidNamespace

Methods

async def unshare(self, flags: CLONE) ‑> NoneType

Expand source code Browse git

async def unshare(self, flags: CLONE) -> None:
    if flags & CLONE.FILES:
        await self.unshare_files()
        flags ^= CLONE.FILES
    if flags:
        await _unshare(self.sysif, flags)

async def setns_user(self, fd: FileDescriptor) ‑> NoneType

Expand source code Browse git

async def setns_user(self, fd: FileDescriptor) -> None:
    # can't setns to a user namespace while sharing CLONE_FS
    await self.unshare(CLONE.FS)
    await self.setns(fd, CLONE.NEWUSER)

async def execveat(self, fd: t.Optional[FileDescriptor], pathname: WrittenPointer[t.Union[str, os.PathLike]], argv: WrittenPointer[ArgList], envp: WrittenPointer[ArgList], flags: AT = AT.NONE, command: Command = None) ‑> NoneType

Expand source code Browse git

async def execveat(self, fd: t.Optional[FileDescriptor],
                   pathname: WrittenPointer[t.Union[str, os.PathLike]],
                   argv: WrittenPointer[ArgList],
                   envp: WrittenPointer[ArgList],
                   flags: AT=AT.NONE,
                   command: Command=None,
) -> None:
    with contextlib.ExitStack() as stack:
        if fd:
            fd_n: t.Optional[rsyscall.near.FileDescriptor] = stack.enter_context(fd.borrow(self))
        else:
            fd_n = None
        stack.enter_context(pathname.borrow(self))
        argv.check_address_space(self)
        envp.check_address_space(self)
        for arg in [*argv.value, *envp.value]:
            stack.enter_context(arg.borrow(self))
        self.manipulating_fd_table = True
        try:
            await _execveat(self.sysif, fd_n, pathname.near, argv.near, envp.near, flags)
        except OSError as exn:
            exn.filename = (fd, pathname.value)
            raise
        finally:
            self.manipulating_fd_table = False
        self._make_fresh_fd_table()
        self._make_fresh_address_space()
        if isinstance(self.process, ChildProcess):
            self.process.did_exec(command)
    await self.sysif.close_interface()

async def execve(self, filename: WrittenPointer[t.Union[str, os.PathLike]], argv: WrittenPointer[ArgList], envp: WrittenPointer[ArgList], command: Command = None) ‑> NoneType

Expand source code Browse git

async def execve(self, filename: WrittenPointer[t.Union[str, os.PathLike]],
                 argv: WrittenPointer[ArgList],
                 envp: WrittenPointer[ArgList],
                 command: Command=None,
) -> None:
    filename.check_address_space(self)
    argv.check_address_space(self)
    envp.check_address_space(self)
    for arg in [*argv.value, *envp.value]:
        arg.check_address_space(self)
    self.manipulating_fd_table = True
    try:
        await _execve(self.sysif, filename.near, argv.near, envp.near)
    except OSError as exn:
        exn.filename = filename.value
        raise
    self.manipulating_fd_table = False
    self._make_fresh_fd_table()
    self._make_fresh_address_space()
    if isinstance(self.process, ChildProcess):
        self.process.did_exec(command)
    await self.sysif.close_interface()

async def exit(self, status: int) ‑> NoneType

Expand source code Browse git

async def exit(self, status: int) -> None:
    self.manipulating_fd_table = True
    await _exit(self.sysif, status)
    self.manipulating_fd_table = False
    self._make_fresh_fd_table()
    # close the syscall interface; we don't have to do this since it'll be
    # GC'd, but maybe we want to be tidy in advance.
    await self.sysif.close_interface()

Inherited members

• EventfdTask:
  • inherit_fd
  • make_fd_handle
  • unshare_files
• SocketTask:
  • socket
• PipeTask:
  • pipe
• PrctlTask:
  • prctl
• ResourceTask:
  • setpriority
• SignalTask:
  • read_oldset_and_check
  • sigmask_block
  • sigprocmask