From 1915fab619a82baf2f986655d87597f793a65c4b Mon Sep 17 00:00:00 2001 From: Chad Harp Date: Tue, 9 Nov 2021 19:24:24 -0600 Subject: [PATCH] tun_darwin (#163) - Remove water and replace with syscalls for tun setup - Support named interfaces - Set up routes with syscalls instead of os/exec Co-authored-by: Wade Simmons --- CHANGELOG.md | 6 +- examples/config.yml | 4 +- tun_darwin.go | 381 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 356 insertions(+), 35 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb4fd08..83d5a67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,10 +33,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 is a large improvement over the TAP driver that was used in previous versions. If you had a previous version of `nebula` running, you will want to disable the tap driver in Control Panel, or uninstall the `tap0901` driver before running this version. (#289) - + - Darwin binaries are now universal (works on both amd64 and arm64), signed, and shipped in a notarized zip file. `nebula-darwin.zip` will be the only darwin release artifact. (#571) +- Darwin uses syscalls and AF_ROUTE to configure the routing table, instead of + using `/sbin/route`. Setting `tun.dev` is now allowed on Darwin as well, it + must be in the format `utun[0-9]+` or it will be ignored. (#163) + ### Deprecated - The `preferred_ranges` option has been supported as a replacement for diff --git a/examples/config.yml b/examples/config.yml index 87b7954..9720ef3 100644 --- a/examples/config.yml +++ b/examples/config.yml @@ -148,7 +148,9 @@ punchy: tun: # When tun is disabled, a lighthouse can be started without a local tun interface (and therefore without root) disabled: false - # Name of the device + # Name of the device. If not set, a default will be chosen by the OS. + # For macOS: if set, must be in the form `utun[0-9]+`. + # For FreeBSD: Required to be set, must be in the form `tun[0-9]+`. dev: nebula1 # Toggles forwarding of local broadcast packets, the address of which depends on the ip/mask encoded in pki.cert drop_local_broadcast: false diff --git a/tun_darwin.go b/tun_darwin.go index 079c80e..0cfd24b 100644 --- a/tun_darwin.go +++ b/tun_darwin.go @@ -7,34 +7,169 @@ import ( "fmt" "io" "net" - "os/exec" - "strconv" + "os" + "syscall" + "unsafe" "github.com/sirupsen/logrus" - "github.com/songgao/water" + netroute "golang.org/x/net/route" + "golang.org/x/sys/unix" ) type Tun struct { + io.ReadWriteCloser Device string Cidr *net.IPNet - MTU int + DefaultMTU int + TXQueueLen int UnsafeRoutes []route l *logrus.Logger - *water.Interface + + // cache out buffer since we need to prepend 4 bytes for tun metadata + out []byte } -func newTun(l *logrus.Logger, deviceName string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) { +type sockaddrCtl struct { + scLen uint8 + scFamily uint8 + ssSysaddr uint16 + scID uint32 + scUnit uint32 + scReserved [5]uint32 +} + +type ifReq struct { + Name [16]byte + Flags uint16 + pad [8]byte +} + +func ioctl(a1, a2, a3 uintptr) error { + _, _, errno := unix.Syscall(unix.SYS_IOCTL, a1, a2, a3) + if errno != 0 { + return errno + } + return nil +} + +var sockaddrCtlSize uintptr = 32 + +const ( + _SYSPROTO_CONTROL = 2 //define SYSPROTO_CONTROL 2 /* kernel control protocol */ + _AF_SYS_CONTROL = 2 //#define AF_SYS_CONTROL 2 /* corresponding sub address type */ + _PF_SYSTEM = unix.AF_SYSTEM //#define PF_SYSTEM AF_SYSTEM + _CTLIOCGINFO = 3227799043 //#define CTLIOCGINFO _IOWR('N', 3, struct ctl_info) + utunControlName = "com.apple.net.utun_control" +) + +type ifreqAddr struct { + Name [16]byte + Addr unix.RawSockaddrInet4 + pad [8]byte +} + +type ifreqMTU struct { + Name [16]byte + MTU int32 + pad [8]byte +} + +type ifreqQLEN struct { + Name [16]byte + Value int32 + pad [8]byte +} + +func newTun(l *logrus.Logger, name string, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int, multiqueue bool) (ifce *Tun, err error) { if len(routes) > 0 { return nil, fmt.Errorf("route MTU not supported in Darwin") } - // NOTE: You cannot set the deviceName under Darwin, so you must check tun.Device after calling .Activate() - return &Tun{ - Cidr: cidr, - MTU: defaultMTU, - UnsafeRoutes: unsafeRoutes, - l: l, - }, nil + ifIndex := -1 + if name != "" && name != "utun" { + _, err := fmt.Sscanf(name, "utun%d", &ifIndex) + if err != nil || ifIndex < 0 { + // NOTE: we don't make this error so we don't break existing + // configs that set a name before it was used. + l.Warn("interface name must be utun[0-9]+ on Darwin, ignoring") + ifIndex = -1 + } + } + + fd, err := unix.Socket(_PF_SYSTEM, unix.SOCK_DGRAM, _SYSPROTO_CONTROL) + if err != nil { + return nil, fmt.Errorf("system socket: %v", err) + } + + var ctlInfo = &struct { + ctlID uint32 + ctlName [96]byte + }{} + + copy(ctlInfo.ctlName[:], []byte(utunControlName)) + + err = ioctl(uintptr(fd), uintptr(_CTLIOCGINFO), uintptr(unsafe.Pointer(ctlInfo))) + if err != nil { + return nil, fmt.Errorf("CTLIOCGINFO: %v", err) + } + + sc := sockaddrCtl{ + scLen: uint8(sockaddrCtlSize), + scFamily: unix.AF_SYSTEM, + ssSysaddr: _AF_SYS_CONTROL, + scID: ctlInfo.ctlID, + scUnit: uint32(ifIndex) + 1, + } + + _, _, errno := unix.RawSyscall( + unix.SYS_CONNECT, + uintptr(fd), + uintptr(unsafe.Pointer(&sc)), + uintptr(sockaddrCtlSize), + ) + if errno != 0 { + return nil, fmt.Errorf("SYS_CONNECT: %v", errno) + } + + var ifName struct { + name [16]byte + } + ifNameSize := uintptr(len(ifName.name)) + _, _, errno = syscall.Syscall6(syscall.SYS_GETSOCKOPT, uintptr(fd), + 2, // SYSPROTO_CONTROL + 2, // UTUN_OPT_IFNAME + uintptr(unsafe.Pointer(&ifName)), + uintptr(unsafe.Pointer(&ifNameSize)), 0) + if errno != 0 { + return nil, fmt.Errorf("SYS_GETSOCKOPT: %v", errno) + } + name = string(ifName.name[:ifNameSize-1]) + + err = syscall.SetNonblock(fd, true) + if err != nil { + return nil, fmt.Errorf("SetNonblock: %v", err) + } + + file := os.NewFile(uintptr(fd), "") + + tun := &Tun{ + ReadWriteCloser: file, + Device: name, + Cidr: cidr, + DefaultMTU: defaultMTU, + TXQueueLen: txQueueLen, + UnsafeRoutes: unsafeRoutes, + l: l, + } + + return tun, nil +} + +func (t *Tun) deviceBytes() (o [16]byte) { + for i, c := range t.Device { + o[i] = byte(c) + } + return } func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU int, routes []route, unsafeRoutes []route, txQueueLen int) (ifce *Tun, err error) { @@ -42,43 +177,223 @@ func newTunFromFd(l *logrus.Logger, deviceFd int, cidr *net.IPNet, defaultMTU in } func (c *Tun) Close() error { - if c.Interface != nil { - return c.Interface.Close() + if c.ReadWriteCloser != nil { + return c.ReadWriteCloser.Close() } return nil } -func (c *Tun) Activate() error { - var err error - c.Interface, err = water.New(water.Config{ - DeviceType: water.TUN, - }) +func (t *Tun) Activate() error { + devName := t.deviceBytes() + + var addr, mask [4]byte + + copy(addr[:], t.Cidr.IP.To4()) + copy(mask[:], t.Cidr.Mask) + + s, err := unix.Socket( + unix.AF_INET, + unix.SOCK_DGRAM, + unix.IPPROTO_IP, + ) + if err != nil { - return fmt.Errorf("activate failed: %v", err) + return err } - c.Device = c.Interface.Name() + fd := uintptr(s) - // TODO use syscalls instead of exec.Command - if err = exec.Command("/sbin/ifconfig", c.Device, c.Cidr.String(), c.Cidr.IP.String()).Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) + ifra := ifreqAddr{ + Name: devName, + Addr: unix.RawSockaddrInet4{ + Family: unix.AF_INET, + Addr: addr, + }, } - if err = exec.Command("/sbin/route", "-n", "add", "-net", c.Cidr.String(), "-interface", c.Device).Run(); err != nil { - return fmt.Errorf("failed to run 'route add': %s", err) + + // Set the device ip address + if err = ioctl(fd, unix.SIOCSIFADDR, uintptr(unsafe.Pointer(&ifra))); err != nil { + return fmt.Errorf("failed to set tun address: %s", err) } - if err = exec.Command("/sbin/ifconfig", c.Device, "mtu", strconv.Itoa(c.MTU)).Run(); err != nil { - return fmt.Errorf("failed to run 'ifconfig': %s", err) + + // Set the device network + ifra.Addr.Addr = mask + if err = ioctl(fd, unix.SIOCSIFNETMASK, uintptr(unsafe.Pointer(&ifra))); err != nil { + return fmt.Errorf("failed to set tun netmask: %s", err) } - // Unsafe path routes - for _, r := range c.UnsafeRoutes { - if err = exec.Command("/sbin/route", "-n", "add", "-net", r.route.String(), "-interface", c.Device).Run(); err != nil { - return fmt.Errorf("failed to run 'route add' for unsafe_route %s: %s", r.route.String(), err) + + // Set the device name + ifrf := ifReq{Name: devName} + if err = ioctl(fd, unix.SIOCGIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + return fmt.Errorf("failed to set tun device name: %s", err) + } + + // Set the MTU on the device + ifm := ifreqMTU{Name: devName, MTU: int32(t.DefaultMTU)} + if err = ioctl(fd, unix.SIOCSIFMTU, uintptr(unsafe.Pointer(&ifm))); err != nil { + return fmt.Errorf("Failed to set tun mtu: %v", err) + } + + /* + // Set the transmit queue length + ifrq := ifreqQLEN{Name: devName, Value: int32(t.TXQueueLen)} + if err = ioctl(fd, unix.SIOCSIFTXQLEN, uintptr(unsafe.Pointer(&ifrq))); err != nil { + // If we can't set the queue length nebula will still work but it may lead to packet loss + l.WithError(err).Error("Failed to set tun tx queue length") } + */ + + // Bring up the interface + ifrf.Flags = ifrf.Flags | unix.IFF_UP + if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + return fmt.Errorf("failed to bring the tun device up: %s", err) + } + + routeSock, err := unix.Socket(unix.AF_ROUTE, unix.SOCK_RAW, unix.AF_UNSPEC) + if err != nil { + return fmt.Errorf("unable to create AF_ROUTE socket: %v", err) + } + defer func() { + unix.Shutdown(routeSock, unix.SHUT_RDWR) + err := unix.Close(routeSock) + if err != nil { + t.l.WithError(err).Error("failed to close AF_ROUTE socket") + } + }() + + routeAddr := &netroute.Inet4Addr{} + maskAddr := &netroute.Inet4Addr{} + linkAddr, err := getLinkAddr(t.Device) + if err != nil { + return err + } + if linkAddr == nil { + return fmt.Errorf("unable to discover link_addr for tun interface") + } + + copy(routeAddr.IP[:], addr[:]) + copy(maskAddr.IP[:], mask[:]) + err = addRoute(routeSock, routeAddr, maskAddr, linkAddr) + if err != nil { + return err + } + + // Run the interface + ifrf.Flags = ifrf.Flags | unix.IFF_UP | unix.IFF_RUNNING + if err = ioctl(fd, unix.SIOCSIFFLAGS, uintptr(unsafe.Pointer(&ifrf))); err != nil { + return fmt.Errorf("failed to run tun device: %s", err) + } + + // Unsafe path routes + for _, r := range t.UnsafeRoutes { + copy(routeAddr.IP[:], r.route.IP.To4()) + copy(maskAddr.IP[:], net.IP(r.route.Mask).To4()) + + err = addRoute(routeSock, routeAddr, maskAddr, linkAddr) + if err != nil { + return err + } + + // TODO how to set metric } return nil } +// Get the LinkAddr for the interface of the given name +// TODO: Is there an easier way to fetch this when we create the interface? +// Maybe SIOCGIFINDEX? but this doesn't appear to exist in the darwin headers. +func getLinkAddr(name string) (*netroute.LinkAddr, error) { + rib, err := netroute.FetchRIB(unix.AF_UNSPEC, unix.NET_RT_IFLIST, 0) + if err != nil { + return nil, err + } + msgs, err := netroute.ParseRIB(unix.NET_RT_IFLIST, rib) + if err != nil { + return nil, err + } + + for _, m := range msgs { + switch m := m.(type) { + case *netroute.InterfaceMessage: + if m.Name == name { + sa, ok := m.Addrs[unix.RTAX_IFP].(*netroute.LinkAddr) + if ok { + return sa, nil + } + } + } + } + + return nil, nil +} + +func addRoute(sock int, addr, mask *netroute.Inet4Addr, link *netroute.LinkAddr) error { + r := netroute.RouteMessage{ + Version: unix.RTM_VERSION, + Type: unix.RTM_ADD, + Flags: unix.RTF_UP, + Seq: 1, + Addrs: []netroute.Addr{ + unix.RTAX_DST: addr, + unix.RTAX_GATEWAY: link, + unix.RTAX_NETMASK: mask, + }, + } + + data, err := r.Marshal() + if err != nil { + return fmt.Errorf("failed to create route.RouteMessage: %v", err) + } + _, err = unix.Write(sock, data[:]) + if err != nil { + return fmt.Errorf("failed to write route.RouteMessage to socket: %v", err) + } + + return nil +} + +var _ io.ReadWriteCloser = (*Tun)(nil) + +func (t *Tun) Read(to []byte) (int, error) { + + buf := make([]byte, len(to)+4) + + n, err := t.ReadWriteCloser.Read(buf) + + copy(to, buf[4:]) + return n - 4, err +} + +// Write is only valid for single threaded use +func (t *Tun) Write(from []byte) (int, error) { + buf := t.out + if cap(buf) < len(from)+4 { + buf = make([]byte, len(from)+4) + t.out = buf + } + buf = buf[:len(from)+4] + + if len(from) == 0 { + return 0, syscall.EIO + } + + // Determine the IP Family for the NULL L2 Header + ipVer := from[0] >> 4 + if ipVer == 4 { + buf[3] = syscall.AF_INET + } else if ipVer == 6 { + buf[3] = syscall.AF_INET6 + } else { + return 0, fmt.Errorf("Unable to determine IP version from packet") + } + + copy(buf[4:], from) + + n, err := t.ReadWriteCloser.Write(buf) + return n - 4, err +} + func (c *Tun) CidrNet() *net.IPNet { return c.Cidr }