From 3b51be63709137b15ba26ab8778c9a8ecca44265 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 2 Nov 2023 19:59:23 -0600 Subject: [PATCH 01/17] Move DHCP library back into Smee: The DHCP library was created in order to provide a mechanism for the DHCP functionality to be written in a very clean and composible way. This was not possible with the previous state of Boots and the personnel involved at that time. The current state of Smee is ready for this functionality to be return and placed in a very clean and composible way into the code base. Moving this functionality back in will improve the development process/experience. It is important to note that we do take on the trade off of not having the combined commit history of the library within Smee. The library GitHub repo will still contain history. Signed-off-by: Jacob Weinstock --- backend/file/file.go | 331 ++++++++++++ backend/file/file_test.go | 372 +++++++++++++ backend/file/testdata/example.yaml | 80 +++ backend/kube/error.go | 7 + backend/kube/index.go | 53 ++ backend/kube/index_test.go | 92 ++++ backend/kube/kube.go | 281 ++++++++++ backend/kube/kube_test.go | 578 ++++++++++++++++++++ backend/noop/noop.go | 23 + backend/noop/noop_test.go | 21 + dhcp/README.md | 22 + dhcp/data/data.go | 116 +++++ dhcp/data/data_test.go | 108 ++++ dhcp/dhcp.go | 112 ++++ dhcp/dhcp_test.go | 116 +++++ dhcp/docs/Backend-File.md | 62 +++ dhcp/docs/Code-Structure.md | 25 + dhcp/docs/Design-Philosophy.md | 88 ++++ dhcp/example/fileBackend/main.go | 76 +++ dhcp/example/kubeBackend/hardware.yaml | 38 ++ dhcp/example/kubeBackend/main.go | 96 ++++ dhcp/handler/handler.go | 19 + dhcp/handler/noop/noop.go | 28 + dhcp/handler/noop/noop_test.go | 45 ++ dhcp/handler/reservation/handler.go | 268 ++++++++++ dhcp/handler/reservation/handler_test.go | 603 +++++++++++++++++++++ dhcp/handler/reservation/option.go | 226 ++++++++ dhcp/handler/reservation/option_test.go | 337 ++++++++++++ dhcp/handler/reservation/reservation.go | 56 ++ dhcp/otel/otel.go | 369 +++++++++++++ dhcp/otel/otel_test.go | 636 +++++++++++++++++++++++ go.sum | 2 + 32 files changed, 5286 insertions(+) create mode 100644 backend/file/file.go create mode 100644 backend/file/file_test.go create mode 100644 backend/file/testdata/example.yaml create mode 100644 backend/kube/error.go create mode 100644 backend/kube/index.go create mode 100644 backend/kube/index_test.go create mode 100644 backend/kube/kube.go create mode 100644 backend/kube/kube_test.go create mode 100644 backend/noop/noop.go create mode 100644 backend/noop/noop_test.go create mode 100644 dhcp/README.md create mode 100644 dhcp/data/data.go create mode 100644 dhcp/data/data_test.go create mode 100644 dhcp/dhcp.go create mode 100644 dhcp/dhcp_test.go create mode 100644 dhcp/docs/Backend-File.md create mode 100644 dhcp/docs/Code-Structure.md create mode 100644 dhcp/docs/Design-Philosophy.md create mode 100644 dhcp/example/fileBackend/main.go create mode 100644 dhcp/example/kubeBackend/hardware.yaml create mode 100644 dhcp/example/kubeBackend/main.go create mode 100644 dhcp/handler/handler.go create mode 100644 dhcp/handler/noop/noop.go create mode 100644 dhcp/handler/noop/noop_test.go create mode 100644 dhcp/handler/reservation/handler.go create mode 100644 dhcp/handler/reservation/handler_test.go create mode 100644 dhcp/handler/reservation/option.go create mode 100644 dhcp/handler/reservation/option_test.go create mode 100644 dhcp/handler/reservation/reservation.go create mode 100644 dhcp/otel/otel.go create mode 100644 dhcp/otel/otel_test.go diff --git a/backend/file/file.go b/backend/file/file.go new file mode 100644 index 00000000..7399b6fe --- /dev/null +++ b/backend/file/file.go @@ -0,0 +1,331 @@ +// Package file watches a file for changes and updates the in memory DHCP data. +package file + +import ( + "context" + "fmt" + "net" + "net/netip" + "net/url" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/fsnotify/fsnotify" + "github.com/ghodss/yaml" + "github.com/go-logr/logr" + "github.com/tinkerbell/smee/dhcp/data" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" +) + +const tracerName = "github.com/tinkerbell/smee/dhcp" + +// Errors used by the file watcher. +var ( + // errFileFormat is returned when the file is not in the correct format, e.g. not valid YAML. + errFileFormat = fmt.Errorf("invalid file format") + errRecordNotFound = fmt.Errorf("record not found") + errParseIP = fmt.Errorf("failed to parse IP from File") + errParseSubnet = fmt.Errorf("failed to parse subnet mask from File") + errParseURL = fmt.Errorf("failed to parse URL") +) + +// netboot is the structure for the data expected in a file. +type netboot struct { + AllowPXE bool `yaml:"allowPxe"` // If true, the client will be provided netboot options in the DHCP offer/ack. + IPXEScriptURL string `yaml:"ipxeScriptUrl"` // Overrides default value of that is passed into DHCP on startup. + IPXEScript string `yaml:"ipxeScript"` // Overrides a default value that is passed into DHCP on startup. + Console string `yaml:"console"` + Facility string `yaml:"facility"` +} + +// dhcp is the structure for the data expected in a file. +type dhcp struct { + MACAddress net.HardwareAddr // The MAC address of the client. + IPAddress string `yaml:"ipAddress"` // yiaddr DHCP header. + SubnetMask string `yaml:"subnetMask"` // DHCP option 1. + DefaultGateway string `yaml:"defaultGateway"` // DHCP option 3. + NameServers []string `yaml:"nameServers"` // DHCP option 6. + Hostname string `yaml:"hostname"` // DHCP option 12. + DomainName string `yaml:"domainName"` // DHCP option 15. + BroadcastAddress string `yaml:"broadcastAddress"` // DHCP option 28. + NTPServers []string `yaml:"ntpServers"` // DHCP option 42. + VLANID string `yaml:"vlanID"` // DHCP option 43.116. + LeaseTime int `yaml:"leaseTime"` // DHCP option 51. + Arch string `yaml:"arch"` // DHCP option 93. + DomainSearch []string `yaml:"domainSearch"` // DHCP option 119. + Netboot netboot `yaml:"netboot"` +} + +// Watcher represents the backend for watching a file for changes and updating the in memory DHCP data. +type Watcher struct { + fileMu sync.RWMutex // protects FilePath for reads + + // FilePath is the path to the file to watch. + FilePath string + + // Log is the logger to be used in the File backend. + Log logr.Logger + dataMu sync.RWMutex // protects data + data []byte // data from file + watcher *fsnotify.Watcher +} + +// NewWatcher creates a new file watcher. +func NewWatcher(l logr.Logger, f string) (*Watcher, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + if err := watcher.Add(f); err != nil { + return nil, err + } + + w := &Watcher{ + FilePath: f, + watcher: watcher, + Log: l, + } + + w.fileMu.RLock() + w.data, err = os.ReadFile(filepath.Clean(f)) + w.fileMu.RUnlock() + if err != nil { + return nil, err + } + + return w, nil +} + +// GetByMac is the implementation of the Backend interface. +// It reads a given file from the in memory data (w.data). +func (w *Watcher) GetByMac(ctx context.Context, mac net.HardwareAddr) (*data.DHCP, *data.Netboot, error) { + tracer := otel.Tracer(tracerName) + _, span := tracer.Start(ctx, "backend.file.GetByMac") + defer span.End() + + // get data from file, translate it, then pass it into setDHCPOpts and setNetworkBootOpts + w.dataMu.RLock() + d := w.data + w.dataMu.RUnlock() + r := make(map[string]dhcp) + if err := yaml.Unmarshal(d, &r); err != nil { + err := fmt.Errorf("%w: %w", err, errFileFormat) + w.Log.Error(err, "failed to unmarshal file data") + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + for k, v := range r { + if strings.EqualFold(k, mac.String()) { + // found a record for this mac address + v.MACAddress = mac + d, n, err := w.translate(v) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + span.SetAttributes(d.EncodeToAttributes()...) + span.SetAttributes(n.EncodeToAttributes()...) + span.SetStatus(codes.Ok, "") + + return d, n, nil + } + } + + err := fmt.Errorf("%w: %s", errRecordNotFound, mac.String()) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err +} + +// GetByIP is the implementation of the Backend interface. +// It reads a given file from the in memory data (w.data). +func (w *Watcher) GetByIP(ctx context.Context, ip net.IP) (*data.DHCP, *data.Netboot, error) { + tracer := otel.Tracer(tracerName) + _, span := tracer.Start(ctx, "backend.file.GetByIP") + defer span.End() + + // get data from file, translate it, then pass it into setDHCPOpts and setNetworkBootOpts + w.dataMu.RLock() + d := w.data + w.dataMu.RUnlock() + r := make(map[string]dhcp) + if err := yaml.Unmarshal(d, &r); err != nil { + err := fmt.Errorf("%w: %w", err, errFileFormat) + w.Log.Error(err, "failed to unmarshal file data") + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + for k, v := range r { + if v.IPAddress == ip.String() { + // found a record for this ip address + v.IPAddress = ip.String() + mac, err := net.ParseMAC(k) + if err != nil { + err := fmt.Errorf("%w: %w", err, errFileFormat) + w.Log.Error(err, "failed to parse mac address") + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + v.MACAddress = mac + d, n, err := w.translate(v) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + span.SetAttributes(d.EncodeToAttributes()...) + span.SetAttributes(n.EncodeToAttributes()...) + span.SetStatus(codes.Ok, "") + + return d, n, nil + } + } + + err := fmt.Errorf("%w: %s", errRecordNotFound, ip.String()) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err +} + +// Start starts watching a file for changes and updates the in memory data (w.data) on changes. +// Start is a blocking method. Use a context cancellation to exit. +func (w *Watcher) Start(ctx context.Context) { + for { + select { + case <-ctx.Done(): + w.Log.Info("stopping watcher") + return + case event, ok := <-w.watcher.Events: + if !ok { + continue + } + if event.Op&fsnotify.Write == fsnotify.Write { + w.Log.Info("file changed, updating cache") + w.fileMu.RLock() + d, err := os.ReadFile(w.FilePath) + w.fileMu.RUnlock() + if err != nil { + w.Log.Error(err, "failed to read file", "file", w.FilePath) + break + } + w.dataMu.Lock() + w.data = d + w.dataMu.Unlock() + } + case err, ok := <-w.watcher.Errors: + if !ok { + continue + } + w.Log.Info("error watching file", "err", err) + } + } +} + +// translate converts the data from the file into a data.DHCP and data.Netboot structs. +func (w *Watcher) translate(r dhcp) (*data.DHCP, *data.Netboot, error) { + d := new(data.DHCP) + n := new(data.Netboot) + + d.MACAddress = r.MACAddress + // ip address, required + ip, err := netip.ParseAddr(r.IPAddress) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", err, errParseIP) + } + d.IPAddress = ip + + // subnet mask, required + sm := net.ParseIP(r.SubnetMask) + if sm == nil { + return nil, nil, errParseSubnet + } + d.SubnetMask = net.IPMask(sm.To4()) + + // default gateway, optional + if dg, err := netip.ParseAddr(r.DefaultGateway); err != nil { + w.Log.Info("failed to parse default gateway", "defaultGateway", r.DefaultGateway, "err", err) + } else { + d.DefaultGateway = dg + } + + // name servers, optional + for _, s := range r.NameServers { + ip := net.ParseIP(s) + if ip == nil { + w.Log.Info("failed to parse name server", "nameServer", s) + break + } + d.NameServers = append(d.NameServers, ip) + } + + // hostname, optional + d.Hostname = r.Hostname + + // domain name, optional + d.DomainName = r.DomainName + + // broadcast address, optional + if ba, err := netip.ParseAddr(r.BroadcastAddress); err != nil { + w.Log.Info("failed to parse broadcast address", "broadcastAddress", r.BroadcastAddress, "err", err) + } else { + d.BroadcastAddress = ba + } + + // ntp servers, optional + for _, s := range r.NTPServers { + ip := net.ParseIP(s) + if ip == nil { + w.Log.Info("failed to parse ntp server", "ntpServer", s) + break + } + d.NTPServers = append(d.NTPServers, ip) + } + + // vlanid + d.VLANID = r.VLANID + + // lease time + d.LeaseTime = uint32(r.LeaseTime) + + // arch + d.Arch = r.Arch + + // domain search + d.DomainSearch = r.DomainSearch + + // allow machine to netboot + n.AllowNetboot = r.Netboot.AllowPXE + + // ipxe script url is optional but if provided, it must be a valid url + if r.Netboot.IPXEScriptURL != "" { + u, err := url.Parse(r.Netboot.IPXEScriptURL) + if err != nil { + return nil, nil, fmt.Errorf("%w: %w", err, errParseURL) + } + n.IPXEScriptURL = u + } + + // ipxe script + if r.Netboot.IPXEScript != "" { + n.IPXEScript = r.Netboot.IPXEScript + } + + // console + if r.Netboot.Console != "" { + n.Console = r.Netboot.Console + } + + // facility + if r.Netboot.Facility != "" { + n.Facility = r.Netboot.Facility + } + + return d, n, nil +} diff --git a/backend/file/file_test.go b/backend/file/file_test.go new file mode 100644 index 00000000..28df9994 --- /dev/null +++ b/backend/file/file_test.go @@ -0,0 +1,372 @@ +package file + +import ( + "bytes" + "context" + "errors" + "fmt" + "io/fs" + "log" + "net" + "net/netip" + "net/url" + "os" + "testing" + "time" + + "github.com/fsnotify/fsnotify" + "github.com/go-logr/logr" + "github.com/go-logr/stdr" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/tinkerbell/smee/dhcp/data" +) + +func TestNewWatcher(t *testing.T) { + tests := map[string]struct { + createFile bool + want string + wantErr error + }{ + "contents equal": {createFile: true, want: "test content here"}, + "file not found": {createFile: false, wantErr: &fs.PathError{}}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + var name string + if tt.createFile { + var err error + name, err = createFile([]byte(tt.want)) + if err != nil { + t.Fatal(err) + } + defer os.Remove(name) + } + w, err := NewWatcher(logr.Discard(), name) + if (err != nil) != (tt.wantErr != nil) { + t.Fatalf("NewWatcher() error = %v; type = %[1]T, wantErr %v; type = %[2]T", err, tt.wantErr) + } + var got string + if tt.wantErr != nil { + got = "" + } else { + got = string(w.data) + } + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func createFile(content []byte) (string, error) { + file, err := os.CreateTemp("", "prefix") + if err != nil { + return "", err + } + defer file.Close() + if _, err := file.Write(content); err != nil { + return "", err + } + return file.Name(), nil +} + +type testData struct { + initial string + after string + action string + expectedOut string +} + +func TestStartAndStop(t *testing.T) { + tt := &testData{action: "cancel", expectedOut: `"level"=0 "msg"="stopping watcher"` + "\n"} + out := &bytes.Buffer{} + l := stdr.New(log.New(out, "", 0)) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + watcher, err := fsnotify.NewWatcher() + if err != nil { + t.Fatal(err) + } + w := &Watcher{Log: l, watcher: watcher} + w.Start(ctx) + if diff := cmp.Diff(out.String(), tt.expectedOut); diff != "" { + t.Fatal(diff) + } +} + +func TestStartFileUpdateError(t *testing.T) { + tt := &testData{expectedOut: `"level"=0 "msg"="file changed, updating cache"` + "\n" + `"msg"="failed to read file" "error"="open not-found.txt: no such file or directory" "file"="not-found.txt"` + "\n" + `"level"=0 "msg"="stopping watcher"` + "\n"} + out := &bytes.Buffer{} + l := stdr.New(log.New(out, "", 0)) + got, name := tt.helper(t, l) + defer os.Remove(name) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-time.After(time.Millisecond) + got.FilePath = "not-found.txt" + got.watcher.Events <- fsnotify.Event{Op: fsnotify.Write} + cancel() + }() + got.Start(ctx) + time.Sleep(time.Second) + if diff := cmp.Diff(out.String(), tt.expectedOut); diff != "" { + t.Fatal(diff) + } +} + +func TestStartFileUpdate(t *testing.T) { + tt := &testData{initial: "once upon a time", after: "\nhello world", expectedOut: "once upon a time\nhello world"} + got, name := tt.helper(t, logr.Discard()) + defer os.Remove(name) + ctx, cancel := context.WithCancel(context.Background()) + go func() { + <-time.After(time.Millisecond) + got.fileMu.Lock() + f, err := os.OpenFile(name, os.O_APPEND|os.O_WRONLY, 0o644) + if err != nil { + t.Log(err) + } + f.Write([]byte(tt.after)) + f.Close() + got.fileMu.Unlock() + time.Sleep(time.Millisecond) + cancel() + }() + got.Start(ctx) + got.dataMu.RLock() + d := got.data + got.dataMu.RUnlock() + if diff := cmp.Diff(string(d), tt.expectedOut); diff != "" { + t.Log(string(d)) + t.Fatal(diff) + } +} + +func TestStartFileUpdateClosedChan(t *testing.T) { + out := &bytes.Buffer{} + l := stdr.New(log.New(out, "", 0)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + watcher, err := fsnotify.NewWatcher() + if err != nil { + t.Fatal(err) + } + w := &Watcher{Log: l, watcher: watcher} + go w.Start(ctx) + close(w.watcher.Events) + time.Sleep(time.Millisecond) + if diff := cmp.Diff(out.String(), ""); diff != "" { + t.Fatal(diff) + } +} + +func TestStartError(t *testing.T) { + tt := &testData{expectedOut: `"level"=0 "msg"="error watching file" "err"="test error"` + "\n" + `"level"=0 "msg"="stopping watcher"` + "\n"} + out := &bytes.Buffer{} + l := stdr.New(log.New(out, "", 0)) + ctx, cancel := context.WithCancel(context.Background()) + watcher, err := fsnotify.NewWatcher() + if err != nil { + t.Fatal(err) + } + w := &Watcher{Log: l, watcher: watcher} + go func() { + time.Sleep(time.Millisecond) + w.watcher.Errors <- fmt.Errorf("test error") + cancel() + }() + w.Start(ctx) + if diff := cmp.Diff(out.String(), tt.expectedOut); diff != "" { + t.Fatal(diff) + } +} + +func TestStartErrorContinue(t *testing.T) { + out := &bytes.Buffer{} + l := stdr.New(log.New(out, "", 0)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + watcher, err := fsnotify.NewWatcher() + if err != nil { + t.Fatal(err) + } + w := &Watcher{Log: l, watcher: watcher} + go w.Start(ctx) + close(w.watcher.Errors) + time.Sleep(time.Millisecond) + if diff := cmp.Diff(out.String(), ""); diff != "" { + t.Fatal(diff) + } +} + +func (tt *testData) helper(t *testing.T, l logr.Logger) (*Watcher, string) { + t.Helper() + name, err := createFile([]byte(tt.initial)) + if err != nil { + t.Fatal(err) + } + w, err := NewWatcher(l, name) + if err != nil { + t.Fatal(err) + } + w.dataMu.RLock() + before := string(w.data) + w.dataMu.RUnlock() + if diff := cmp.Diff(before, tt.initial); diff != "" { + t.Fatal("before", diff) + } + + return w, name +} + +func TestTranslate(t *testing.T) { + input := dhcp{ + MACAddress: []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05}, + IPAddress: "192.168.2.150", + SubnetMask: "255.255.255.0", + DefaultGateway: "192.168.2.1", + NameServers: []string{"1.1.1.1", "8.8.8.8"}, + Hostname: "test-server", + DomainName: "example.com", + BroadcastAddress: "192.168.2.255", + NTPServers: []string{"132.163.96.2"}, + VLANID: "100", + LeaseTime: 86400, + Arch: "x86_64", + DomainSearch: []string{"example.com"}, + Netboot: netboot{ + AllowPXE: true, + IPXEScriptURL: "http://boot.netboot.xyz", + IPXEScript: "#!ipxe\nchain http://boot.netboot.xyz", + Console: "ttyS0", + Facility: "onprem", + }, + } + wantDHCP := &data.DHCP{ + MACAddress: []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05}, + IPAddress: netip.MustParseAddr("192.168.2.150"), + SubnetMask: net.IPv4Mask(255, 255, 255, 0), + DefaultGateway: netip.MustParseAddr("192.168.2.1"), + NameServers: []net.IP{{1, 1, 1, 1}, {8, 8, 8, 8}}, + Hostname: "test-server", + DomainName: "example.com", + BroadcastAddress: netip.MustParseAddr("192.168.2.255"), + NTPServers: []net.IP{{132, 163, 96, 2}}, + VLANID: "100", + LeaseTime: 86400, + Arch: "x86_64", + DomainSearch: []string{"example.com"}, + } + wantNetboot := &data.Netboot{ + AllowNetboot: true, + IPXEScriptURL: &url.URL{Scheme: "http", Host: "boot.netboot.xyz"}, + IPXEScript: "#!ipxe\nchain http://boot.netboot.xyz", + Console: "ttyS0", + Facility: "onprem", + } + w := &Watcher{Log: logr.Discard()} + gotDHCP, gotNetboot, err := w.translate(input) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff(gotDHCP, wantDHCP, cmpopts.IgnoreUnexported(netip.Addr{})); diff != "" { + t.Error(diff) + } + if diff := cmp.Diff(gotNetboot, wantNetboot); diff != "" { + t.Error(diff) + } +} + +func TestTranslateErrors(t *testing.T) { + tests := map[string]struct { + input dhcp + wantErr error + }{ + "invalid IP": {input: dhcp{IPAddress: "not an IP"}, wantErr: errParseIP}, + "invalid subnet mask": {input: dhcp{IPAddress: "1.1.1.1", SubnetMask: "not a mask"}, wantErr: errParseSubnet}, + "invalid gateway": {input: dhcp{IPAddress: "1.1.1.1", SubnetMask: "192.168.1.255", DefaultGateway: "not a gateway"}, wantErr: nil}, + "invalid broadcast address": {input: dhcp{IPAddress: "1.1.1.1", SubnetMask: "192.168.1.255"}, wantErr: nil}, + "invalid NameServers": {input: dhcp{IPAddress: "1.1.1.1", SubnetMask: "192.168.1.255", NameServers: []string{"no good"}}, wantErr: nil}, + "invalid ntpservers": {input: dhcp{IPAddress: "1.1.1.1", SubnetMask: "192.168.1.255", NTPServers: []string{"no good"}}, wantErr: nil}, + "invalid ipxe script url": {input: dhcp{IPAddress: "1.1.1.1", SubnetMask: "255.255.255.0", Netboot: netboot{IPXEScriptURL: ":not a url"}}, wantErr: errParseURL}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + w := &Watcher{Log: stdr.New(log.New(os.Stdout, "", log.Lshortfile))} + if _, _, err := w.translate(tt.input); !errors.Is(err, tt.wantErr) { + t.Errorf("translate() = %T, want %T", err, tt.wantErr) + } + }) + } +} + +func TestGetByMac(t *testing.T) { + tests := map[string]struct { + mac net.HardwareAddr + badData bool + wantErr error + }{ + "no record found": {mac: net.HardwareAddr{0x00, 0x01, 0x02, 0x03, 0x04, 0x05}, wantErr: errRecordNotFound}, + "record found": {mac: net.HardwareAddr{0x08, 0x00, 0x27, 0x29, 0x4e, 0x67}, wantErr: nil}, + "fail error translating": {mac: net.HardwareAddr{0x08, 0x00, 0x27, 0x29, 0x4e, 0x68}, wantErr: errParseIP}, + "fail parsing file": {badData: true, wantErr: errFileFormat}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + data := "testdata/example.yaml" + if tt.badData { + var err error + data, err = createFile([]byte("not a yaml file")) + if err != nil { + t.Fatal(err) + } + defer os.Remove(data) + } + w, err := NewWatcher(logr.Discard(), data) + if err != nil { + t.Fatal(err) + } + _, _, err = w.GetByMac(context.Background(), tt.mac) + if !errors.Is(err, tt.wantErr) { + t.Fatal(err) + } + }) + } +} + +func TestGetByIP(t *testing.T) { + tests := map[string]struct { + ip net.IP + badData bool + wantErr error + }{ + "no record found": {ip: net.IPv4(172, 168, 2, 1), wantErr: errRecordNotFound}, + "record found": {ip: net.IPv4(192, 168, 2, 153), wantErr: nil}, + "fail parsing file": {badData: true, wantErr: errFileFormat}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + data := "testdata/example.yaml" + if tt.badData { + var err error + data, err = createFile([]byte("not a yaml file")) + if err != nil { + t.Fatal(err) + } + defer os.Remove(data) + } + w, err := NewWatcher(logr.Discard(), data) + if err != nil { + t.Fatal(err) + } + _, _, err = w.GetByIP(context.Background(), tt.ip) + if !errors.Is(err, tt.wantErr) { + t.Fatal(err) + } + }) + } +} diff --git a/backend/file/testdata/example.yaml b/backend/file/testdata/example.yaml new file mode 100644 index 00000000..0708ff05 --- /dev/null +++ b/backend/file/testdata/example.yaml @@ -0,0 +1,80 @@ +--- +08:00:27:29:4E:67: + ipAddress: '192.168.2.153' + subnetMask: '255.255.255.0' + defaultGateway: '192.168.2.1' + nameServers: + - '8.8.8.8' + - '1.1.1.1' + hostname: 'pxe-virtualbox' + domainName: 'example.com' + broadcastAddress: '192.168.2.255' + ntpServers: + - '132.163.96.2' + - '132.163.96.3' + leaseTime: 86400 + domainSearch: + - 'example.com' + netboot: + allowPxe: true + ipxeScriptUrl: 'https://boot.netboot.xyz' +52:54:00:aa:88:2a: + ipAddress: '192.168.2.15' + subnetMask: '255.255.255.0' + defaultGateway: '192.168.2.1' + nameServers: + - '8.8.8.8' + - '1.1.1.1' + hostname: 'sandbox' + domainName: 'example.com' + broadcastAddress: '192.168.2.255' + ntpServers: + - '132.163.96.2' + - '132.163.96.3' + leaseTime: 86400 + domainSearch: + - 'example.com' + netboot: + allowPxe: true + ipxeScriptUrl: 'https://boot.netboot.xyz' +86:96:b0:6e:ca:36: + ipAddress: '192.168.2.158' + subnetMask: '255.255.255.0' + defaultGateway: '192.168.2.1' + nameServers: + - '8.8.8.8' + - '1.1.1.1' + hostname: 'pxe-proxmox' + domainName: 'example.com' + broadcastAddress: '192.168.2.255' + ntpServers: + - '132.163.96.2' + - '132.163.96.3' + leaseTime: 86400 + domainSearch: + - 'example.com' + netboot: + allowPxe: true + ipxeScriptUrl: 'http://boot.netboot.xyz' +b4:96:91:6f:33:d0: + ipAddress: '192.168.56.15' + subnetMask: '255.255.255.0' + defaultGateway: '192.168.56.4' + nameServers: + - '8.8.8.8' + - '1.1.1.1' + hostname: 'dhcp-testing' + domainName: 'example.com' + broadcastAddress: '192.168.56.255' + ntpServers: + - '132.163.96.2' + - '132.163.96.3' + leaseTime: 86400 + domainSearch: + - 'example.com' + netboot: + allowPxe: true + ipxeScriptUrl: 'https://boot.netboot.xyz' +08:00:27:29:4E:68: # bad data + ipAddress: '3' + subnetMask: '255.255.255.0' \ No newline at end of file diff --git a/backend/kube/error.go b/backend/kube/error.go new file mode 100644 index 00000000..4bd8b1e7 --- /dev/null +++ b/backend/kube/error.go @@ -0,0 +1,7 @@ +package kube + +type hardwareNotFoundError struct{} + +func (hardwareNotFoundError) NotFound() bool { return true } + +func (hardwareNotFoundError) Error() string { return "hardware not found" } diff --git a/backend/kube/index.go b/backend/kube/index.go new file mode 100644 index 00000000..7032a374 --- /dev/null +++ b/backend/kube/index.go @@ -0,0 +1,53 @@ +package kube + +import ( + "github.com/tinkerbell/tink/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MACAddrIndex is an index used with a controller-runtime client to lookup hardware by MAC. +const MACAddrIndex = ".Spec.Interfaces.MAC" + +// MACAddrs returns a list of MAC addresses for a Hardware object. +func MACAddrs(obj client.Object) []string { + hw, ok := obj.(*v1alpha1.Hardware) + if !ok { + return nil + } + return GetMACs(hw) +} + +// GetMACs retrieves all MACs associated with h. +func GetMACs(h *v1alpha1.Hardware) []string { + var macs []string + for _, i := range h.Spec.Interfaces { + if i.DHCP != nil && i.DHCP.MAC != "" { + macs = append(macs, i.DHCP.MAC) + } + } + + return macs +} + +// IPAddrIndex is an index used with a controller-runtime client to lookup hardware by IP. +const IPAddrIndex = ".Spec.Interfaces.DHCP.IP" + +// IPAddrs returns a list of IP addresses for a Hardware object. +func IPAddrs(obj client.Object) []string { + hw, ok := obj.(*v1alpha1.Hardware) + if !ok { + return nil + } + return GetIPs(hw) +} + +// GetIPs retrieves all IP addresses. +func GetIPs(h *v1alpha1.Hardware) []string { + var ips []string + for _, i := range h.Spec.Interfaces { + if i.DHCP != nil && i.DHCP.IP != nil && i.DHCP.IP.Address != "" { + ips = append(ips, i.DHCP.IP.Address) + } + } + return ips +} diff --git a/backend/kube/index_test.go b/backend/kube/index_test.go new file mode 100644 index 00000000..7b7590d6 --- /dev/null +++ b/backend/kube/index_test.go @@ -0,0 +1,92 @@ +package kube + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/tinkerbell/tink/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestMACAddrs(t *testing.T) { + tests := map[string]struct { + hw client.Object + want []string + }{ + "not a v1alpha1.Hardware object": {hw: &v1alpha1.Workflow{}, want: nil}, + "2 MACs": {hw: &v1alpha1.Hardware{ + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + DHCP: &v1alpha1.DHCP{ + MAC: "00:00:00:00:00:00", + }, + }, + { + DHCP: &v1alpha1.DHCP{ + MAC: "00:00:00:00:00:01", + }, + }, + { + DHCP: &v1alpha1.DHCP{}, + }, + }, + }, + }, want: []string{"00:00:00:00:00:00", "00:00:00:00:00:01"}}, + "no interfaces": {hw: &v1alpha1.Hardware{}, want: nil}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + macs := MACAddrs(tc.hw) + if diff := cmp.Diff(macs, tc.want); diff != "" { + t.Errorf("unexpected MACs (+want -got):\n%s", diff) + } + }) + } +} + +func TestIPAddrs(t *testing.T) { + tests := map[string]struct { + hw client.Object + want []string + }{ + "not a v1alpha1.Hardware object": {hw: &v1alpha1.Workflow{}, want: nil}, + "2 IPs": {hw: &v1alpha1.Hardware{ + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + DHCP: &v1alpha1.DHCP{ + IP: &v1alpha1.IP{ + Address: "192.168.2.1", + }, + }, + }, + { + DHCP: &v1alpha1.DHCP{ + IP: &v1alpha1.IP{ + Address: "192.168.2.2", + }, + }, + }, + { + DHCP: &v1alpha1.DHCP{}, + }, + { + DHCP: &v1alpha1.DHCP{ + IP: &v1alpha1.IP{}, + }, + }, + }, + }, + }, want: []string{"192.168.2.1", "192.168.2.2"}}, + "no interfaces": {hw: &v1alpha1.Hardware{}, want: nil}, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got := IPAddrs(tc.hw) + if diff := cmp.Diff(tc.want, got); diff != "" { + t.Errorf("unexpected IPs (-want +got):\n%s", diff) + } + }) + } +} diff --git a/backend/kube/kube.go b/backend/kube/kube.go new file mode 100644 index 00000000..8940a47b --- /dev/null +++ b/backend/kube/kube.go @@ -0,0 +1,281 @@ +// Package kube is a backend implementation that uses the Tinkerbell CRDs to get DHCP data. +package kube + +import ( + "context" + "errors" + "fmt" + "net" + "net/netip" + "net/url" + + "github.com/tinkerbell/smee/dhcp/data" + "github.com/tinkerbell/tink/api/v1alpha1" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +const tracerName = "github.com/tinkerbell/smee/dhcp" + +// Backend is a backend implementation that uses the Tinkerbell CRDs to get DHCP data. +type Backend struct { + cluster cluster.Cluster +} + +// NewBackend returns a controller-runtime cluster.Cluster with the Tinkerbell runtime +// scheme registered, and indexers for: +// * Hardware by MAC address +// * Hardware by IP address +// +// Callers must instantiate the client-side cache by calling Start() before use. +func NewBackend(conf *rest.Config, opts ...cluster.Option) (*Backend, error) { + rs := runtime.NewScheme() + + if err := scheme.AddToScheme(rs); err != nil { + return nil, err + } + + if err := v1alpha1.AddToScheme(rs); err != nil { + return nil, err + } + + opts = append([]cluster.Option{func(o *cluster.Options) { o.Scheme = rs }}, opts...) + o := []cluster.Option{func(o *cluster.Options) { o.Scheme = rs }} + o = append(o, opts...) + c, err := cluster.New(conf, o...) + if err != nil { + return nil, fmt.Errorf("failed to create new cluster config: %w", err) + } + + if err := c.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.Hardware{}, MACAddrIndex, MACAddrs); err != nil { + return nil, fmt.Errorf("failed to setup indexer: %w", err) + } + + if err := c.GetFieldIndexer().IndexField(context.Background(), &v1alpha1.Hardware{}, IPAddrIndex, IPAddrs); err != nil { + return nil, fmt.Errorf("failed to setup indexer(.spec.interfaces.dhcp.ip.address): %w", err) + } + + return &Backend{cluster: c}, nil +} + +// Start starts the client-side cache. +func (b *Backend) Start(ctx context.Context) error { + return b.cluster.Start(ctx) +} + +// GetByMac implements the handler.BackendReader interface and returns DHCP and netboot data based on a mac address. +func (b *Backend) GetByMac(ctx context.Context, mac net.HardwareAddr) (*data.DHCP, *data.Netboot, error) { + tracer := otel.Tracer(tracerName) + ctx, span := tracer.Start(ctx, "backend.kube.GetByMac") + defer span.End() + hardwareList := &v1alpha1.HardwareList{} + + if err := b.cluster.GetClient().List(ctx, hardwareList, &client.MatchingFields{MACAddrIndex: mac.String()}); err != nil { + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, fmt.Errorf("failed listing hardware for (%v): %w", mac, err) + } + + if len(hardwareList.Items) == 0 { + err := hardwareNotFoundError{} + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + + if len(hardwareList.Items) > 1 { + err := fmt.Errorf("got %d hardware objects for mac %s, expected only 1", len(hardwareList.Items), mac) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + + i := v1alpha1.Interface{} + for _, iface := range hardwareList.Items[0].Spec.Interfaces { + if iface.DHCP.MAC == mac.String() { + i = iface + break + } + } + + d, err := toDHCPData(i.DHCP) + if err != nil { + err = fmt.Errorf("failed to convert hardware to DHCP data: %w", err) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + n, err := toNetbootData(i.Netboot) + if err != nil { + err = fmt.Errorf("failed to convert hardware to netboot data: %w", err) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + + span.SetAttributes(d.EncodeToAttributes()...) + span.SetAttributes(n.EncodeToAttributes()...) + span.SetStatus(codes.Ok, "") + + return d, n, nil +} + +// GetByIP implements the handler.BackendReader interface and returns DHCP and netboot data based on an IP address. +func (b *Backend) GetByIP(ctx context.Context, ip net.IP) (*data.DHCP, *data.Netboot, error) { + tracer := otel.Tracer(tracerName) + ctx, span := tracer.Start(ctx, "backend.kube.GetByIP") + defer span.End() + hardwareList := &v1alpha1.HardwareList{} + + if err := b.cluster.GetClient().List(ctx, hardwareList, &client.MatchingFields{IPAddrIndex: ip.String()}); err != nil { + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, fmt.Errorf("failed listing hardware for (%v): %w", ip, err) + } + + if len(hardwareList.Items) == 0 { + err := hardwareNotFoundError{} + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + + if len(hardwareList.Items) > 1 { + err := fmt.Errorf("got %d hardware objects for ip: %s, expected only 1", len(hardwareList.Items), ip) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + + i := v1alpha1.Interface{} + for _, iface := range hardwareList.Items[0].Spec.Interfaces { + if iface.DHCP.IP.Address == ip.String() { + i = iface + break + } + } + + d, err := toDHCPData(i.DHCP) + if err != nil { + err = fmt.Errorf("failed to convert hardware to DHCP data: %w", err) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + n, err := toNetbootData(i.Netboot) + if err != nil { + err = fmt.Errorf("failed to convert hardware to netboot data: %w", err) + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + + span.SetAttributes(d.EncodeToAttributes()...) + span.SetAttributes(n.EncodeToAttributes()...) + span.SetStatus(codes.Ok, "") + + return d, n, nil +} + +// toDHCPData converts a v1alpha1.DHCP to a data.DHCP data structure. +// if required fields are missing, an error is returned. +// Required fields: v1alpha1.Interface.DHCP.MAC, v1alpha1.Interface.DHCP.IP.Address, v1alpha1.Interface.DHCP.IP.Netmask. +func toDHCPData(h *v1alpha1.DHCP) (*data.DHCP, error) { + if h == nil { + return nil, errors.New("no DHCP data") + } + d := new(data.DHCP) + + var err error + // MACAddress is required + if d.MACAddress, err = net.ParseMAC(h.MAC); err != nil { + return nil, err + } + + if h.IP != nil { + // IPAddress is required + if d.IPAddress, err = netip.ParseAddr(h.IP.Address); err != nil { + return nil, err + } + // Netmask is required + sm := net.ParseIP(h.IP.Netmask) + if sm == nil { + return nil, errors.New("no netmask") + } + d.SubnetMask = net.IPMask(sm.To4()) + } else { + return nil, errors.New("no IP data") + } + + // Gateway is optional, but should be a valid IP address if present + if h.IP.Gateway != "" { + if d.DefaultGateway, err = netip.ParseAddr(h.IP.Gateway); err != nil { + return nil, err + } + } + + // name servers, optional + for _, s := range h.NameServers { + ip := net.ParseIP(s) + if ip == nil { + break + } + d.NameServers = append(d.NameServers, ip) + } + + // hostname, optional + d.Hostname = h.Hostname + + // lease time required + d.LeaseTime = uint32(h.LeaseTime) + + // arch + d.Arch = h.Arch + + // vlanid + d.VLANID = h.VLANID + + return d, nil +} + +// toNetbootData converts a hardware interface to a data.Netboot data structure. +func toNetbootData(i *v1alpha1.Netboot) (*data.Netboot, error) { + if i == nil { + return nil, errors.New("no netboot data") + } + n := new(data.Netboot) + + // allow machine to netboot + if i.AllowPXE != nil { + n.AllowNetboot = *i.AllowPXE + } + + // ipxe script url is optional but if provided, it must be a valid url + if i.IPXE != nil { + if i.IPXE.URL != "" { + u, err := url.ParseRequestURI(i.IPXE.URL) + if err != nil { + return nil, err + } + n.IPXEScriptURL = u + } + } + + // ipxescript + if i.IPXE != nil { + n.IPXEScript = i.IPXE.Contents + } + + // console + n.Console = "" + + // facility + n.Facility = "" + + return n, nil +} diff --git a/backend/kube/kube_test.go b/backend/kube/kube_test.go new file mode 100644 index 00000000..e02902d8 --- /dev/null +++ b/backend/kube/kube_test.go @@ -0,0 +1,578 @@ +package kube + +import ( + "context" + "net" + "net/http" + "net/netip" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/tinkerbell/smee/dhcp/data" + "github.com/tinkerbell/tink/api/v1alpha1" + "k8s.io/apimachinery/pkg/api/meta" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/rest" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/cache/informertest" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/cluster" +) + +func TestNewBackend(t *testing.T) { + tests := map[string]struct { + conf *rest.Config + opt cluster.Option + shouldErr bool + }{ + "no config": {shouldErr: true}, + "failed index field": {shouldErr: true, conf: new(rest.Config), opt: func(o *cluster.Options) { + cl := fake.NewClientBuilder().Build() + o.NewClient = func(config *rest.Config, options client.Options) (client.Client, error) { + return cl, nil + } + o.MapperProvider = func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + return cl.RESTMapper(), nil + } + }}, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + b, err := NewBackend(tt.conf, tt.opt) + if tt.shouldErr && err == nil { + t.Fatal("expected error") + } + if !tt.shouldErr && err != nil { + t.Fatal(err) + } + if !tt.shouldErr && b == nil { + t.Fatal("expected backend") + } + }) + } +} + +func TestToDHCPData(t *testing.T) { + tests := map[string]struct { + in *v1alpha1.DHCP + want *data.DHCP + shouldErr bool + }{ + "nil input": { + in: nil, + shouldErr: true, + }, + "no mac": { + in: &v1alpha1.DHCP{}, + shouldErr: true, + }, + "bad mac": { + in: &v1alpha1.DHCP{MAC: "bad"}, + shouldErr: true, + }, + "no ip": { + in: &v1alpha1.DHCP{MAC: "aa:bb:cc:dd:ee:ff", IP: &v1alpha1.IP{}}, + shouldErr: true, + }, + "no subnet": { + in: &v1alpha1.DHCP{MAC: "aa:bb:cc:dd:ee:ff", IP: &v1alpha1.IP{Address: "192.168.2.4"}}, + shouldErr: true, + }, + "v1alpha1.IP == nil": { + in: &v1alpha1.DHCP{MAC: "aa:bb:cc:dd:ee:ff", IP: nil}, + shouldErr: true, + }, + "bad gateway": { + in: &v1alpha1.DHCP{MAC: "aa:bb:cc:dd:ee:ff", IP: &v1alpha1.IP{Address: "192.168.2.4", Netmask: "255.255.254.0", Gateway: "bad"}}, + shouldErr: true, + }, + "one bad nameserver": { + in: &v1alpha1.DHCP{ + MAC: "00:00:00:00:00:04", + NameServers: []string{"1.1.1.1", "bad"}, + IP: &v1alpha1.IP{ + Address: "192.168.2.4", + Netmask: "255.255.0.0", + Gateway: "192.168.2.1", + }, + }, + want: &data.DHCP{ + SubnetMask: net.IPv4Mask(255, 255, 0, 0), + DefaultGateway: netip.MustParseAddr("192.168.2.1"), + NameServers: []net.IP{net.IPv4(1, 1, 1, 1)}, + IPAddress: netip.MustParseAddr("192.168.2.4"), + MACAddress: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x04}, + }, + }, + "full": { + in: &v1alpha1.DHCP{ + MAC: "00:00:00:00:00:04", + Hostname: "test", + LeaseTime: 3600, + NameServers: []string{"1.1.1.1"}, + IP: &v1alpha1.IP{ + Address: "192.168.1.4", + Netmask: "255.255.255.0", + Gateway: "192.168.1.1", + }, + }, + want: &data.DHCP{ + SubnetMask: net.IPv4Mask(255, 255, 255, 0), + DefaultGateway: netip.MustParseAddr("192.168.1.1"), + NameServers: []net.IP{net.IPv4(1, 1, 1, 1)}, + Hostname: "test", + LeaseTime: 3600, + IPAddress: netip.MustParseAddr("192.168.1.4"), + MACAddress: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x04}, + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := toDHCPData(tt.in) + if tt.shouldErr && err == nil { + t.Fatal("expected error") + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(netip.Addr{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestToNetbootData(t *testing.T) { + tests := map[string]struct { + in *v1alpha1.Netboot + want *data.Netboot + shouldErr bool + }{ + "nil input": {in: nil, shouldErr: true}, + "bad ipxe url": {in: &v1alpha1.Netboot{IPXE: &v1alpha1.IPXE{URL: "bad"}}, shouldErr: true}, + "successful": {in: &v1alpha1.Netboot{IPXE: &v1alpha1.IPXE{URL: "http://example.com/ipxe.ipxe"}}, want: &data.Netboot{IPXEScriptURL: &url.URL{Scheme: "http", Host: "example.com", Path: "/ipxe.ipxe"}}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := toNetbootData(tt.in) + if tt.shouldErr && err == nil { + t.Fatal("expected error") + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(netip.Addr{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestGetByIP(t *testing.T) { + tests := map[string]struct { + hwObject []v1alpha1.Hardware + wantDHCP *data.DHCP + wantNetboot *data.Netboot + shouldErr bool + failToList bool + }{ + "empty hardware list": {shouldErr: true, hwObject: []v1alpha1.Hardware{}}, + "more than one hardware": {shouldErr: true, hwObject: []v1alpha1.Hardware{hwObject1, hwObject2}}, + "bad dhcp data": {shouldErr: true, hwObject: []v1alpha1.Hardware{badDHCPObject2}}, + "bad netboot data": {shouldErr: true, hwObject: []v1alpha1.Hardware{badNetbootObject2}}, + "fail to list hardware": {shouldErr: true, failToList: true}, + "good data": {hwObject: []v1alpha1.Hardware{hwObject1}, wantDHCP: &data.DHCP{ + MACAddress: net.HardwareAddr{0x3c, 0xec, 0xef, 0x4c, 0x4f, 0x54}, + IPAddress: netip.MustParseAddr("172.16.10.100"), + SubnetMask: []byte{0xff, 0xff, 0xff, 0x00}, + DefaultGateway: netip.MustParseAddr("255.255.255.0"), + NameServers: []net.IP{ + {0x1, 0x1, 0x1, 0x1}, + }, + Hostname: "sm01", + LeaseTime: 86400, + Arch: "x86_64", + }, wantNetboot: &data.Netboot{ + AllowNetboot: true, + IPXEScriptURL: &url.URL{ + Scheme: "http", + Host: "netboot.xyz", + }, + }}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + rs := runtime.NewScheme() + if err := scheme.AddToScheme(rs); err != nil { + t.Fatal(err) + } + if err := v1alpha1.AddToScheme(rs); err != nil { + t.Fatal(err) + } + + ct := fake.NewClientBuilder() + if !tc.failToList { + ct = ct.WithScheme(rs) + ct = ct.WithRuntimeObjects(&v1alpha1.HardwareList{}) + ct = ct.WithIndex(&v1alpha1.Hardware{}, IPAddrIndex, func(obj client.Object) []string { + var list []string + for _, elem := range tc.hwObject { + list = append(list, elem.Spec.Interfaces[0].DHCP.IP.Address) + } + return list + }) + } + if len(tc.hwObject) > 0 { + t.Logf("%+v", tc.hwObject[0].Spec.Interfaces[0].DHCP) + t.Logf("%+v", tc.hwObject[0].Spec.Interfaces[0].DHCP.IP) + ct = ct.WithLists(&v1alpha1.HardwareList{Items: tc.hwObject}) + } + cl := ct.Build() + + fn := func(o *cluster.Options) { + o.NewClient = func(config *rest.Config, options client.Options) (client.Client, error) { + return cl, nil + } + o.MapperProvider = func(_ *rest.Config, _ *http.Client) (meta.RESTMapper, error) { + return cl.RESTMapper(), nil + } + o.NewCache = func(config *rest.Config, options cache.Options) (cache.Cache, error) { + return &informertest.FakeInformers{Scheme: cl.Scheme()}, nil + } + } + rc := new(rest.Config) + b, err := NewBackend(rc, fn) + if err != nil { + t.Fatal(err) + } + + go b.Start(context.Background()) + gotDHCP, gotNetboot, err := b.GetByIP(context.Background(), net.IPv4(172, 16, 10, 100)) + if tc.shouldErr && err == nil { + t.Log(err) + t.Fatal("expected error") + } + + if diff := cmp.Diff(gotDHCP, tc.wantDHCP, cmpopts.IgnoreUnexported(netip.Addr{})); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff(gotNetboot, tc.wantNetboot); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestGetByMac(t *testing.T) { + tests := map[string]struct { + hwObject []v1alpha1.Hardware + wantDHCP *data.DHCP + wantNetboot *data.Netboot + shouldErr bool + failToList bool + }{ + "empty hardware list": {shouldErr: true}, + "more than one hardware": {shouldErr: true, hwObject: []v1alpha1.Hardware{hwObject1, hwObject2}}, + "bad dhcp data": {shouldErr: true, hwObject: []v1alpha1.Hardware{badDHCPObject}}, + "bad netboot data": {shouldErr: true, hwObject: []v1alpha1.Hardware{badNetbootObject}}, + "fail to list hardware": {shouldErr: true, failToList: true}, + "good data": {hwObject: []v1alpha1.Hardware{hwObject1}, wantDHCP: &data.DHCP{ + MACAddress: net.HardwareAddr{0x3c, 0xec, 0xef, 0x4c, 0x4f, 0x54}, + IPAddress: netip.MustParseAddr("172.16.10.100"), + SubnetMask: []byte{0xff, 0xff, 0xff, 0x00}, + DefaultGateway: netip.MustParseAddr("255.255.255.0"), + NameServers: []net.IP{ + {0x1, 0x1, 0x1, 0x1}, + }, + Hostname: "sm01", + LeaseTime: 86400, + Arch: "x86_64", + }, wantNetboot: &data.Netboot{ + AllowNetboot: true, + IPXEScriptURL: &url.URL{ + Scheme: "http", + Host: "netboot.xyz", + }, + }}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + rs := runtime.NewScheme() + if err := scheme.AddToScheme(rs); err != nil { + t.Fatal(err) + } + if err := v1alpha1.AddToScheme(rs); err != nil { + t.Fatal(err) + } + + ct := fake.NewClientBuilder() + if !tc.failToList { + ct = ct.WithScheme(rs) + ct = ct.WithRuntimeObjects(&v1alpha1.HardwareList{}) + ct = ct.WithIndex(&v1alpha1.Hardware{}, MACAddrIndex, func(obj client.Object) []string { + var list []string + for _, elem := range tc.hwObject { + list = append(list, elem.Spec.Interfaces[0].DHCP.MAC) + } + return list + }) + } + if len(tc.hwObject) > 0 { + t.Logf("%+v", tc.hwObject[0].Spec.Interfaces[0].DHCP) + t.Logf("%+v", tc.hwObject[0].Spec.Interfaces[0].DHCP.MAC) + ct = ct.WithLists(&v1alpha1.HardwareList{Items: tc.hwObject}) + } + cl := ct.Build() + + fn := func(o *cluster.Options) { + o.NewClient = func(config *rest.Config, options client.Options) (client.Client, error) { + return cl, nil + } + o.MapperProvider = func(c *rest.Config, httpClient *http.Client) (meta.RESTMapper, error) { + return cl.RESTMapper(), nil + } + o.NewCache = func(config *rest.Config, options cache.Options) (cache.Cache, error) { + return &informertest.FakeInformers{Scheme: cl.Scheme()}, nil + } + } + rc := new(rest.Config) + b, err := NewBackend(rc, fn) + if err != nil { + t.Fatal(err) + } + + go b.Start(context.Background()) + gotDHCP, gotNetboot, err := b.GetByMac(context.Background(), net.HardwareAddr{0x3c, 0xec, 0xef, 0x4c, 0x4f, 0x54}) + if tc.shouldErr && err == nil { + t.Log(err) + t.Fatal("expected error") + } + + if diff := cmp.Diff(gotDHCP, tc.wantDHCP, cmpopts.IgnoreUnexported(netip.Addr{})); diff != "" { + t.Fatal(diff) + } + + if diff := cmp.Diff(gotNetboot, tc.wantNetboot); diff != "" { + t.Fatal(diff) + } + }) + } +} + +var hwObject1 = v1alpha1.Hardware{ + TypeMeta: v1.TypeMeta{ + Kind: "Hardware", + APIVersion: "tinkerbell.org/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "machine1", + Namespace: "default", + }, + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + Netboot: &v1alpha1.Netboot{ + AllowPXE: &[]bool{true}[0], + AllowWorkflow: &[]bool{true}[0], + IPXE: &v1alpha1.IPXE{ + URL: "http://netboot.xyz", + }, + }, + DHCP: &v1alpha1.DHCP{ + Arch: "x86_64", + Hostname: "sm01", + IP: &v1alpha1.IP{ + Address: "172.16.10.100", + Gateway: "172.16.10.1", + Netmask: "255.255.255.0", + }, + LeaseTime: 86400, + MAC: "3c:ec:ef:4c:4f:54", + NameServers: []string{"1.1.1.1"}, + UEFI: true, + }, + }, + }, + }, +} + +var hwObject2 = v1alpha1.Hardware{ + TypeMeta: v1.TypeMeta{ + Kind: "Hardware", + APIVersion: "tinkerbell.org/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "machine2", + Namespace: "default", + }, + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + Netboot: &v1alpha1.Netboot{ + AllowPXE: &[]bool{true}[0], + AllowWorkflow: &[]bool{true}[0], + IPXE: &v1alpha1.IPXE{ + URL: "http://netboot.xyz", + }, + }, + DHCP: &v1alpha1.DHCP{ + Arch: "x86_64", + Hostname: "sm01", + IP: &v1alpha1.IP{ + Address: "172.16.10.101", + Gateway: "172.16.10.1", + Netmask: "255.255.255.0", + }, + LeaseTime: 86400, + MAC: "3c:ec:ef:4c:4f:55", + NameServers: []string{"1.1.1.1"}, + UEFI: true, + }, + }, + }, + }, +} + +var badDHCPObject = v1alpha1.Hardware{ + TypeMeta: v1.TypeMeta{ + Kind: "Hardware", + APIVersion: "tinkerbell.org/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "machine2", + Namespace: "default", + }, + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + Netboot: &v1alpha1.Netboot{ + AllowPXE: &[]bool{true}[0], + AllowWorkflow: &[]bool{true}[0], + IPXE: &v1alpha1.IPXE{ + URL: "http://netboot.xyz", + }, + }, + DHCP: &v1alpha1.DHCP{ + Arch: "x86_64", + Hostname: "sm01", + IP: &v1alpha1.IP{ + Address: "172.16.10.100", + Gateway: "bad-address", + Netmask: "255.255.255.0", + }, + LeaseTime: 86400, + MAC: "3c:ec:ef:4c:4f:54", + NameServers: []string{"1.1.1.1"}, + UEFI: true, + }, + }, + }, + }, +} + +var badDHCPObject2 = v1alpha1.Hardware{ + TypeMeta: v1.TypeMeta{ + Kind: "Hardware", + APIVersion: "tinkerbell.org/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "machine2", + Namespace: "default", + }, + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + Netboot: &v1alpha1.Netboot{ + AllowPXE: &[]bool{true}[0], + AllowWorkflow: &[]bool{true}[0], + IPXE: &v1alpha1.IPXE{ + URL: "http://netboot.xyz", + }, + }, + DHCP: &v1alpha1.DHCP{ + Arch: "x86_64", + Hostname: "sm01", + IP: &v1alpha1.IP{ + Address: "172.16.10.100", + Gateway: "bad-address", + Netmask: "255.255.255.0", + }, + LeaseTime: 86400, + MAC: "3c:ec:ef:4c:4f:55", + NameServers: []string{"1.1.1.1"}, + UEFI: true, + }, + }, + }, + }, +} + +var badNetbootObject = v1alpha1.Hardware{ + TypeMeta: v1.TypeMeta{ + Kind: "Hardware", + APIVersion: "tinkerbell.org/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "machine2", + Namespace: "default", + }, + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + Netboot: &v1alpha1.Netboot{ + IPXE: &v1alpha1.IPXE{ + URL: "bad-url", + }, + }, + DHCP: &v1alpha1.DHCP{ + Hostname: "sm01", + IP: &v1alpha1.IP{ + Address: "172.16.10.101", + Gateway: "172.16.10.1", + Netmask: "255.255.255.0", + }, + LeaseTime: 86400, + MAC: "3c:ec:ef:4c:4f:54", + NameServers: []string{"1.1.1.1"}, + }, + }, + }, + }, +} + +var badNetbootObject2 = v1alpha1.Hardware{ + TypeMeta: v1.TypeMeta{ + Kind: "Hardware", + APIVersion: "tinkerbell.org/v1alpha1", + }, + ObjectMeta: v1.ObjectMeta{ + Name: "machine2", + Namespace: "default", + }, + Spec: v1alpha1.HardwareSpec{ + Interfaces: []v1alpha1.Interface{ + { + Netboot: &v1alpha1.Netboot{ + IPXE: &v1alpha1.IPXE{ + URL: "bad-url", + }, + }, + DHCP: &v1alpha1.DHCP{ + Hostname: "sm01", + IP: &v1alpha1.IP{ + Address: "172.16.10.100", + Gateway: "172.16.10.1", + Netmask: "255.255.255.0", + }, + LeaseTime: 86400, + MAC: "3c:ec:ef:4c:4f:54", + NameServers: []string{"1.1.1.1"}, + }, + }, + }, + }, +} diff --git a/backend/noop/noop.go b/backend/noop/noop.go new file mode 100644 index 00000000..37fb3baf --- /dev/null +++ b/backend/noop/noop.go @@ -0,0 +1,23 @@ +// Package noop is a backend handler that does nothing. +package noop + +import ( + "context" + "errors" + "net" + + "github.com/tinkerbell/smee/dhcp/data" +) + +// Handler is a noop backend. +type Handler struct{} + +// GetByMac returns an error. +func (h Handler) GetByMac(_ context.Context, _ net.HardwareAddr) (*data.DHCP, *data.Netboot, error) { + return nil, nil, errors.New("no backend specified, please specify a backend") +} + +// GetByIP returns an error. +func (h Handler) GetByIP(_ context.Context, _ net.IP) (*data.DHCP, *data.Netboot, error) { + return nil, nil, errors.New("no backend specified, please specify a backend") +} diff --git a/backend/noop/noop_test.go b/backend/noop/noop_test.go new file mode 100644 index 00000000..cda5e60a --- /dev/null +++ b/backend/noop/noop_test.go @@ -0,0 +1,21 @@ +package noop + +import ( + "context" + "errors" + "testing" + + "github.com/google/go-cmp/cmp" +) + +func TestNoop(t *testing.T) { + want := errors.New("no backend specified, please specify a backend") + _, _, got := Handler{}.GetByMac(context.TODO(), nil) + if diff := cmp.Diff(want.Error(), got.Error()); diff != "" { + t.Fatal(diff) + } + _, _, got = Handler{}.GetByIP(context.TODO(), nil) + if diff := cmp.Diff(want.Error(), got.Error()); diff != "" { + t.Fatal(diff) + } +} diff --git a/dhcp/README.md b/dhcp/README.md new file mode 100644 index 00000000..a28e58b8 --- /dev/null +++ b/dhcp/README.md @@ -0,0 +1,22 @@ +# dhcp + +DHCP library with multiple backends. All IP addresses are served as DHCP reservations. There are no lease pools as are normally found in DHCP servers. + +## Backends + +- [Tink Kubernetes CRDs](https://github.com/tinkerbell/tink/blob/main/config/crd/bases/tinkerbell.org_hardware.yaml) + - This backend is also the main use case. + It pulls hardware data from Kubernetes CRDs for use in serving DHCP clients. +- [File based](./docs/Backend-File.md) + - This backend is for mainly for testing and development. + It reads a file for hardware data to use in serving DHCP clients. + See [example.yaml](../backend/file/testdata/example.yaml) for the data model. + +## Definitions + +**DHCP Reservation:** +A fixed IP address that is reserved for a specific client. + +**DHCP Lease:** +An IP address, that can potentially change, that is assigned to a client by the DHCP server. +The IP is typically pulled from a pool or subnet of available IP addresses. diff --git a/dhcp/data/data.go b/dhcp/data/data.go new file mode 100644 index 00000000..7fd0fccd --- /dev/null +++ b/dhcp/data/data.go @@ -0,0 +1,116 @@ +// Package data is an interface between DHCP backend implementations and the DHCP server. +package data + +import ( + "net" + "net/netip" + "net/url" + "strings" + + "github.com/insomniacslk/dhcp/dhcpv4" + "go.opentelemetry.io/otel/attribute" +) + +// Packet holds the data that is passed to a DHCP handler. +type Packet struct { + // Peer is the address of the client that sent the DHCP message. + Peer net.Addr + // Pkt is the DHCP message. + Pkt *dhcpv4.DHCPv4 + // Md is the metadata that was passed to the DHCP server. + Md *Metadata +} + +// Metadata holds metadata about the DHCP packet that was received. +type Metadata struct { + // IfName is the name of the interface that the DHCP message was received on. + IfName string + // IfIndex is the index of the interface that the DHCP message was received on. + IfIndex int +} + +// DHCP holds the DHCP headers and options to be set in a DHCP handler response. +// This is the API between a DHCP handler and a backend. +type DHCP struct { + MACAddress net.HardwareAddr // chaddr DHCP header. + IPAddress netip.Addr // yiaddr DHCP header. + SubnetMask net.IPMask // DHCP option 1. + DefaultGateway netip.Addr // DHCP option 3. + NameServers []net.IP // DHCP option 6. + Hostname string // DHCP option 12. + DomainName string // DHCP option 15. + BroadcastAddress netip.Addr // DHCP option 28. + NTPServers []net.IP // DHCP option 42. + VLANID string // DHCP option 43.116. + LeaseTime uint32 // DHCP option 51. + Arch string // DHCP option 93. + DomainSearch []string // DHCP option 119. +} + +// Netboot holds info used in netbooting a client. +type Netboot struct { + AllowNetboot bool // If true, the client will be provided netboot options in the DHCP offer/ack. + IPXEScriptURL *url.URL // Overrides a default value that is passed into DHCP on startup. + IPXEScript string // Overrides a default value that is passed into DHCP on startup. + Console string + Facility string +} + +// EncodeToAttributes returns a slice of opentelemetry attributes that can be used to set span.SetAttributes. +func (d *DHCP) EncodeToAttributes() []attribute.KeyValue { + var ns []string + for _, e := range d.NameServers { + ns = append(ns, e.String()) + } + + var ntp []string + for _, e := range d.NTPServers { + ntp = append(ntp, e.String()) + } + + var ip string + if d.IPAddress.Compare(netip.Addr{}) != 0 { + ip = d.IPAddress.String() + } + + var sm string + if d.SubnetMask != nil { + sm = net.IP(d.SubnetMask).String() + } + + var dfg string + if d.DefaultGateway.Compare(netip.Addr{}) != 0 { + dfg = d.DefaultGateway.String() + } + + var ba string + if d.BroadcastAddress.Compare(netip.Addr{}) != 0 { + ba = d.BroadcastAddress.String() + } + + return []attribute.KeyValue{ + attribute.String("DHCP.MACAddress", d.MACAddress.String()), + attribute.String("DHCP.IPAddress", ip), + attribute.String("DHCP.SubnetMask", sm), + attribute.String("DHCP.DefaultGateway", dfg), + attribute.String("DHCP.NameServers", strings.Join(ns, ",")), + attribute.String("DHCP.Hostname", d.Hostname), + attribute.String("DHCP.DomainName", d.DomainName), + attribute.String("DHCP.BroadcastAddress", ba), + attribute.String("DHCP.NTPServers", strings.Join(ntp, ",")), + attribute.Int64("DHCP.LeaseTime", int64(d.LeaseTime)), + attribute.String("DHCP.DomainSearch", strings.Join(d.DomainSearch, ",")), + } +} + +// EncodeToAttributes returns a slice of opentelemetry attributes that can be used to set span.SetAttributes. +func (n *Netboot) EncodeToAttributes() []attribute.KeyValue { + var s string + if n.IPXEScriptURL != nil { + s = n.IPXEScriptURL.String() + } + return []attribute.KeyValue{ + attribute.Bool("Netboot.AllowNetboot", n.AllowNetboot), + attribute.String("Netboot.IPXEScriptURL", s), + } +} diff --git a/dhcp/data/data_test.go b/dhcp/data/data_test.go new file mode 100644 index 00000000..80dff906 --- /dev/null +++ b/dhcp/data/data_test.go @@ -0,0 +1,108 @@ +package data + +import ( + "net" + "net/netip" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "go.opentelemetry.io/otel/attribute" +) + +func TestDHCPEncodeToAttributes(t *testing.T) { + tests := map[string]struct { + dhcp *DHCP + want []attribute.KeyValue + }{ + "successful encode of zero value DHCP struct": { + dhcp: &DHCP{}, + want: []attribute.KeyValue{ + attribute.String("DHCP.MACAddress", ""), + attribute.String("DHCP.IPAddress", ""), + attribute.String("DHCP.Hostname", ""), + attribute.String("DHCP.SubnetMask", ""), + attribute.String("DHCP.DefaultGateway", ""), + attribute.String("DHCP.NameServers", ""), + attribute.String("DHCP.DomainName", ""), + attribute.String("DHCP.BroadcastAddress", ""), + attribute.String("DHCP.NTPServers", ""), + attribute.Int64("DHCP.LeaseTime", 0), + attribute.String("DHCP.DomainSearch", ""), + }, + }, + "successful encode of populated DHCP struct": { + dhcp: &DHCP{ + MACAddress: []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05}, + IPAddress: netip.MustParseAddr("192.168.2.150"), + SubnetMask: []byte{255, 255, 255, 0}, + DefaultGateway: netip.MustParseAddr("192.168.2.1"), + NameServers: []net.IP{{1, 1, 1, 1}, {8, 8, 8, 8}}, + Hostname: "test", + DomainName: "example.com", + BroadcastAddress: netip.MustParseAddr("192.168.2.255"), + NTPServers: []net.IP{{132, 163, 96, 2}}, + LeaseTime: 86400, + DomainSearch: []string{"example.com", "example.org"}, + }, + want: []attribute.KeyValue{ + attribute.String("DHCP.MACAddress", "00:01:02:03:04:05"), + attribute.String("DHCP.IPAddress", "192.168.2.150"), + attribute.String("DHCP.Hostname", "test"), + attribute.String("DHCP.SubnetMask", "255.255.255.0"), + attribute.String("DHCP.DefaultGateway", "192.168.2.1"), + attribute.String("DHCP.NameServers", "1.1.1.1,8.8.8.8"), + attribute.String("DHCP.DomainName", "example.com"), + attribute.String("DHCP.BroadcastAddress", "192.168.2.255"), + attribute.String("DHCP.NTPServers", "132.163.96.2"), + attribute.Int64("DHCP.LeaseTime", 86400), + attribute.String("DHCP.DomainSearch", "example.com,example.org"), + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + want := attribute.NewSet(tt.want...) + got := attribute.NewSet(tt.dhcp.EncodeToAttributes()...) + enc := attribute.DefaultEncoder() + if diff := cmp.Diff(got.Encoded(enc), want.Encoded(enc)); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestNetbootEncodeToAttributes(t *testing.T) { + tests := map[string]struct { + netboot *Netboot + want []attribute.KeyValue + }{ + "successful encode of zero value Netboot struct": { + netboot: &Netboot{}, + want: []attribute.KeyValue{ + attribute.Bool("Netboot.AllowNetboot", false), + attribute.String("Netboot.IPXEScriptURL", ""), + }, + }, + "successful encode of populated Netboot struct": { + netboot: &Netboot{ + AllowNetboot: true, + IPXEScriptURL: &url.URL{Scheme: "http", Host: "example.com"}, + }, + want: []attribute.KeyValue{ + attribute.Bool("Netboot.AllowNetboot", true), + attribute.String("Netboot.IPXEScriptURL", "http://example.com"), + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + want := attribute.NewSet(tt.want...) + got := attribute.NewSet(tt.netboot.EncodeToAttributes()...) + enc := attribute.DefaultEncoder() + if diff := cmp.Diff(got.Encoded(enc), want.Encoded(enc)); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/dhcp/dhcp.go b/dhcp/dhcp.go new file mode 100644 index 00000000..aaeaddc7 --- /dev/null +++ b/dhcp/dhcp.go @@ -0,0 +1,112 @@ +// Package dhcp providers UDP listening and serving functionality. +package dhcp + +import ( + "context" + "net" + + "github.com/go-logr/logr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/dhcpv4/server4" + "github.com/tinkerbell/smee/dhcp/data" + "golang.org/x/net/ipv4" +) + +// Handler is a type that defines the handler function to be called every time a +// valid DHCPv4 message is received +// type Handler func(ctx context.Context, conn net.PacketConn, d data.Packet). +type Handler interface { + Handle(ctx context.Context, conn *ipv4.PacketConn, d data.Packet) +} + +// Server represents a DHCPv4 server object. +type Server struct { + Conn net.PacketConn + Handlers []Handler + Logger logr.Logger +} + +// Serve serves requests. +func (s *Server) Serve(ctx context.Context) error { + go func() { + <-ctx.Done() + _ = s.Close() + }() + s.Logger.Info("Server listening on", "addr", s.Conn.LocalAddr()) + + nConn := ipv4.NewPacketConn(s.Conn) + if err := nConn.SetControlMessage(ipv4.FlagInterface, true); err != nil { + s.Logger.Info("error setting control message", "err", err) + return err + } + + defer func() { + _ = nConn.Close() + }() + for { + // Max UDP packet size is 65535. Max DHCPv4 packet size is 576. An ethernet frame is 1500 bytes. + // We use 4096 as a reasonable buffer size. dhcpv4.FromBytes will handle the rest. + rbuf := make([]byte, 4096) + n, cm, peer, err := nConn.ReadFrom(rbuf) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + s.Logger.Info("error reading from packet conn", "err", err) + return err + } + + m, err := dhcpv4.FromBytes(rbuf[:n]) + if err != nil { + s.Logger.Info("error parsing DHCPv4 request", "err", err) + continue + } + + upeer, ok := peer.(*net.UDPAddr) + if !ok { + s.Logger.Info("not a UDP connection? Peer is", "peer", peer) + continue + } + // Set peer to broadcast if the client did not have an IP. + if upeer.IP == nil || upeer.IP.To4().Equal(net.IPv4zero) { + upeer = &net.UDPAddr{ + IP: net.IPv4bcast, + Port: upeer.Port, + } + } + + var ifName string + if n, err := net.InterfaceByIndex(cm.IfIndex); err == nil { + ifName = n.Name + } + + for _, handler := range s.Handlers { + go handler.Handle(ctx, nConn, data.Packet{Peer: upeer, Pkt: m, Md: &data.Metadata{IfName: ifName, IfIndex: cm.IfIndex}}) + } + } +} + +// Close sends a termination request to the server, and closes the UDP listener. +func (s *Server) Close() error { + return s.Conn.Close() +} + +// NewServer initializes and returns a new Server object. +func NewServer(ifname string, addr *net.UDPAddr, handler ...Handler) (*Server, error) { + s := &Server{ + Handlers: handler, + Logger: logr.Discard(), + } + + if s.Conn == nil { + var err error + conn, err := server4.NewIPv4UDPConn(ifname, addr) + if err != nil { + return nil, err + } + s.Conn = conn + } + return s, nil +} diff --git a/dhcp/dhcp_test.go b/dhcp/dhcp_test.go new file mode 100644 index 00000000..ef6dead1 --- /dev/null +++ b/dhcp/dhcp_test.go @@ -0,0 +1,116 @@ +package dhcp + +import ( + "context" + "net" + "net/netip" + "testing" + + "github.com/go-logr/logr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/dhcpv4/nclient4" + "github.com/tinkerbell/smee/dhcp/data" + "golang.org/x/net/ipv4" + "golang.org/x/net/nettest" +) + +type mock struct { + Log logr.Logger + ServerIP net.IP + LeaseTime uint32 + YourIP net.IP + NameServers []net.IP + SubnetMask net.IPMask + Router net.IP +} + +func (m *mock) Handle(_ context.Context, conn *ipv4.PacketConn, d data.Packet) { + if m.Log.GetSink() == nil { + m.Log = logr.Discard() + } + + mods := m.setOpts() + switch mt := d.Pkt.MessageType(); mt { + case dhcpv4.MessageTypeDiscover: + mods = append(mods, dhcpv4.WithMessageType(dhcpv4.MessageTypeOffer)) + case dhcpv4.MessageTypeRequest: + mods = append(mods, dhcpv4.WithMessageType(dhcpv4.MessageTypeAck)) + case dhcpv4.MessageTypeRelease: + mods = append(mods, dhcpv4.WithMessageType(dhcpv4.MessageTypeAck)) + default: + m.Log.Info("unsupported message type", "type", mt.String()) + return + } + reply, err := dhcpv4.NewReplyFromRequest(d.Pkt, mods...) + if err != nil { + m.Log.Error(err, "error creating reply") + return + } + cm := &ipv4.ControlMessage{IfIndex: d.Md.IfIndex} + if _, err := conn.WriteTo(reply.ToBytes(), cm, d.Peer); err != nil { + m.Log.Error(err, "failed to send reply") + return + } + m.Log.Info("sent reply") +} + +func (m *mock) setOpts() []dhcpv4.Modifier { + mods := []dhcpv4.Modifier{ + dhcpv4.WithGeneric(dhcpv4.OptionServerIdentifier, m.ServerIP), + dhcpv4.WithServerIP(m.ServerIP), + dhcpv4.WithLeaseTime(m.LeaseTime), + dhcpv4.WithYourIP(m.YourIP), + dhcpv4.WithDNS(m.NameServers...), + dhcpv4.WithNetmask(m.SubnetMask), + dhcpv4.WithRouter(m.Router), + } + + return mods +} + +func dhcp(ctx context.Context) (*dhcpv4.DHCPv4, error) { + rifs, err := nettest.RoutedInterface("ip", net.FlagUp|net.FlagBroadcast) + if err != nil { + return nil, err + } + c, err := nclient4.New(rifs.Name, + nclient4.WithServerAddr(&net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 7676}), + nclient4.WithUnicast(&net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 7677}), + ) + if err != nil { + return nil, err + } + defer c.Close() + + return c.DiscoverOffer(ctx) +} + +func TestServe(t *testing.T) { + tests := map[string]struct { + h Handler + addr netip.AddrPort + }{ + "success": {addr: netip.MustParseAddrPort("127.0.0.1:7676"), h: &mock{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + s, err := NewServer("lo", net.UDPAddrFromAddrPort(tt.addr), tt.h) + if err != nil { + t.Fatal(err) + } + ctx, done := context.WithCancel(context.Background()) + defer done() + + go s.Serve(ctx) + + // make client calls + d, err := dhcp(ctx) + if err != nil { + t.Fatal(err) + } + t.Log(d) + + done() + }) + } +} diff --git a/dhcp/docs/Backend-File.md b/dhcp/docs/Backend-File.md new file mode 100644 index 00000000..a4f9b8ac --- /dev/null +++ b/dhcp/docs/Backend-File.md @@ -0,0 +1,62 @@ +# File Watcher Backend + +This document gives an overview of the file watcher backend. +This backend will read in and watch a file on disk for changes. +The data from this file will then be used for serving DHCP requests. + +## Why + +This backend exists mainly for testing and development. +It allows the DHCP server to be run without having to spin up any additional backend servers, like [Tink](https://github.com/tinkerbell/tink) or [Cacher](https://github.com/packethost/cacher). + +## Usage + +```bash +# See the file example/main.go for details on how to select and use this backend in code. +go run example/main.go +``` + +Below is an example of the format used for this file watcher backend. +See this [example.yaml](../backend/file/testdata/example.yaml) for a full working example of the data model. + +```yaml +--- +08:00:27:29:4E:67: + ipAddress: '192.168.2.153' + subnetMask: '255.255.255.0' + defaultGateway: '192.168.2.1' + nameServers: + - '8.8.8.8' + - '1.1.1.1' + hostname: 'pxe-virtualbox' + domainName: 'example.com' + broadcastAddress: '192.168.2.255' + ntpServers: + - '132.163.96.2' + - '132.163.96.3' + leaseTime: 86400 + domainSearch: + - 'example.com' + netboot: + allowPxe: true + ipxeScriptUrl: 'https://boot.netboot.xyz' +52:54:00:aa:88:2a: + ipAddress: '192.168.2.15' + subnetMask: '255.255.255.0' + defaultGateway: '192.168.2.1' + nameServers: + - '8.8.8.8' + - '1.1.1.1' + hostname: 'sandbox' + domainName: 'example.com' + broadcastAddress: '192.168.2.255' + ntpServers: + - '132.163.96.2' + - '132.163.96.3' + leaseTime: 86400 + domainSearch: + - 'example.com' + netboot: + allowPxe: true + ipxeScriptUrl: 'https://boot.netboot.xyz' +``` diff --git a/dhcp/docs/Code-Structure.md b/dhcp/docs/Code-Structure.md new file mode 100644 index 00000000..4fb87c16 --- /dev/null +++ b/dhcp/docs/Code-Structure.md @@ -0,0 +1,25 @@ +# Code Structure + +## Backend + +Responsible for communicating with an external persistence source and returning data from said source. +Backends live in the `backend/` directory. + +## Handler + +Responsible for reading a DHCP packet from a source, calling a backend, and responding to the source. +All business logic for responding or reacting to DHCP messages lives here. +Handlers live in the `handler/` directory. + +## Listener + +Responsible for listening for UDP packets on the specified address and port. +A default listener can be used. + +## Server + +Responsible for filtering for DHCP packets received by the listener and calling the specified handler. + +## Functional description + +Server(listener, handler(backend)) diff --git a/dhcp/docs/Design-Philosophy.md b/dhcp/docs/Design-Philosophy.md new file mode 100644 index 00000000..1a97a4bd --- /dev/null +++ b/dhcp/docs/Design-Philosophy.md @@ -0,0 +1,88 @@ +# Design Philosophy + +This living document describes some Go design philosophies we endeavor to incorporate when working, building, or writing in Go. + +## General + +1. Prefer easy to understand over easy to do +2. First do it, then do it right, then do it better, then make it testable [14] +3. When you spawn goroutines, make it clear when - or whether - they exit. [2] +4. Packages that are imported only for their side effects should be avoided [4] +5. Package level and global variables should be avoided +6. magic is bad; global state is magic → no package level vars; no func init [13] + +## Dependencies + +1. External dependencies should be tried and fail fast or just keep trying + - For example, external connections, port binding, environment variables, secrets, etc + - Examples of "failing fast" + - Try external connections immediately + - Binding to ports immediately + - Examples of "keep trying" + - Block ingress traffic or calls until external connections are successful + - Should be accompanied by some way to check health status of external connections +2. Make all dependencies explicit [11] + +## Naming + +1. Naming general rules [12] + - Structs are plain nouns: API, Replica, Object + - Interfaces are active nouns: Reader, Writer, JobProcessor + - Functions and methods are verbs: Read, Process, Sync +2. Package names [15] + - Short: no more than one word + - No plural + - Lower case + - Informative about the service it provides + - Avoid packages named utility/utilities or model/models +3. Avoid renaming imports except to avoid a name collision; good package names should not require renaming [3] + +## Interfaces + +1. Accept interfaces, return structs [5] +2. Small interfaces are better [6] +3. Define an interface when you actually need it, not when you foresee needing it [7] +4. Interfaces [15] + - Use interfaces as function/method arguments & as field types + - Small interfaces are better + +## Functions/Methods + +1. All top-level, exported names should have doc comments, as should non-trivial unexported type or function declarations. [1] +2. Methods/functions [15] + - One function has one goal + - Simple names + - Reduce the number of nesting levels +3. Only func main has the right to decide which flags, env variables, config files are available to the user [10a],[10b] +4. `context.Context` should, in most cases, be the first argument of all functions or methods +5. Prefer synchronous functions - functions which return their results directly or finish any callbacks or channel ops before returning - over asynchronous ones. [8] + +## Errors + +1. Error Handling [15] + - Func `main` should normally be the only one calling fatal errors or `os.Exit` + +## Source files + +1. One file should be named like the package [9] +2. One file = One responsibility [9] +3. If you only have one command prefer a top level `main.go`, if you have more than one command put them in a `cmd/` package + +--- + +[1]: https://github.com/golang/go/wiki/CodeReviewComments#doc-comments +[2]: https://github.com/golang/go/wiki/CodeReviewComments#goroutine-lifetimes +[3]: https://github.com/golang/go/wiki/CodeReviewComments#imports +[4]: https://github.com/golang/go/wiki/CodeReviewComments#import-blank +[5]: https://medium.com/@cep21/what-accept-interfaces-return-structs-means-in-go-2fe879e25ee8 +[6]: https://www.practical-go-lessons.com/chap-40-design-recommendations?s=03#use-interfaces +[7]: http://c2.com/xp/YouArentGonnaNeedIt.html +[8]: https://github.com/golang/go/wiki/CodeReviewComments#synchronous-functions +[9]: https://www.practical-go-lessons.com/chap-40-design-recommendations?s=03#source-files +[10a]: https://thoughtbot.com/blog/where-to-define-command-line-flags-in-go +[10b]: https://peter.bourgon.org/go-best-practices-2016/#configuration +[11]: https://peter.bourgon.org/go-best-practices-2016/#top-tip-9 +[12]: https://twitter.com/peterbourgon/status/1121023995107782656 +[13]: https://peter.bourgon.org/blog/2017/06/09/theory-of-modern-go.html +[14]: https://code.tutsplus.com/articles/master-developers-addy-osmani--net-31661 +[15]: https://www.practical-go-lessons.com/chap-40-design-recommendations?s=03#key-takeaways diff --git a/dhcp/example/fileBackend/main.go b/dhcp/example/fileBackend/main.go new file mode 100644 index 00000000..9f86f14d --- /dev/null +++ b/dhcp/example/fileBackend/main.go @@ -0,0 +1,76 @@ +// package main is an example of how to use the dhcp package with the file backend. +package main + +import ( + "context" + "log" + "net" + "net/netip" + "net/url" + "os" + "os/signal" + "syscall" + + "github.com/equinix-labs/otel-init-go/otelinit" + "github.com/go-logr/logr" + "github.com/go-logr/stdr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/dhcpv4/server4" + "github.com/tinkerbell/smee/backend/file" + "github.com/tinkerbell/smee/dhcp" + "github.com/tinkerbell/smee/dhcp/handler" + "github.com/tinkerbell/smee/dhcp/handler/reservation" +) + +func main() { + ctx, done := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGHUP, syscall.SIGTERM) + defer done() + ctx, otelShutdown := otelinit.InitOpenTelemetry(ctx, "github.com/tinkerbell/smee/dhcp") + defer otelShutdown(ctx) + + l := stdr.New(log.New(os.Stdout, "", log.Lshortfile)) + l = l.WithName("github.com/tinkerbell/smee/dhcp") + // 1. create the backend + // 2. create the handler(backend) + // 3. create the listener(handler) + backend, err := fileBackend(ctx, l, "./backend/file/testdata/example.yaml") + if err != nil { + panic(err) + } + + h := &reservation.Handler{ + Log: l, + IPAddr: netip.MustParseAddr("192.168.2.225"), + Netboot: reservation.Netboot{ + IPXEBinServerTFTP: netip.MustParseAddrPort("192.168.1.34:69"), + IPXEBinServerHTTP: &url.URL{Scheme: "http", Host: "192.168.1.34:8080"}, + IPXEScriptURL: func(*dhcpv4.DHCPv4) *url.URL { + return &url.URL{Scheme: "https", Host: "boot.netboot.xyz"} + }, + Enabled: true, + }, + OTELEnabled: true, + Backend: backend, + } + conn, err := server4.NewIPv4UDPConn("", net.UDPAddrFromAddrPort(netip.MustParseAddrPort("0.0.0.0:67"))) + if err != nil { + panic(err) + } + + defer func() { + _ = conn.Close() + }() + server := &dhcp.Server{Logger: l, Conn: conn, Handlers: []dhcp.Handler{h}} + l.Info("starting server", "addr", h.IPAddr) + l.Error(server.Serve(ctx), "done") + l.Info("done") +} + +func fileBackend(ctx context.Context, l logr.Logger, f string) (handler.BackendReader, error) { + fb, err := file.NewWatcher(l, f) + if err != nil { + return nil, err + } + go fb.Start(ctx) + return fb, nil +} diff --git a/dhcp/example/kubeBackend/hardware.yaml b/dhcp/example/kubeBackend/hardware.yaml new file mode 100644 index 00000000..bbcfad29 --- /dev/null +++ b/dhcp/example/kubeBackend/hardware.yaml @@ -0,0 +1,38 @@ +apiVersion: "tinkerbell.org/v1alpha1" +kind: Hardware +metadata: + name: sm01 + namespace: default +spec: + disks: + - device: /dev/nvme0n1 + metadata: + facility: + facility_code: onprem + manufacturer: + slug: supermicro + instance: + userdata: "" + hostname: "sm01" + id: "de:ad:c0:de:ca:fe" + operating_system: + distro: "ubuntu" + os_slug: "ubuntu_20_04" + version: "20.04" + interfaces: + - dhcp: + arch: x86_64 + hostname: sm01 + ip: + address: 192.168.2.17 + gateway: 192.169.2.1 + netmask: 255.255.255.0 + lease_time: 86400 + mac: de:ad:c0:de:ca:fe + name_servers: + - 192.168.2.1 + - 10.1.1.11 + uefi: true + netboot: + allowPXE: true + allowWorkflow: true diff --git a/dhcp/example/kubeBackend/main.go b/dhcp/example/kubeBackend/main.go new file mode 100644 index 00000000..4f4d252c --- /dev/null +++ b/dhcp/example/kubeBackend/main.go @@ -0,0 +1,96 @@ +// package main is an example of how to use the dhcp package with the kube backend. +package main + +import ( + "context" + "log" + "net" + "net/netip" + "net/url" + "os" + "os/signal" + "syscall" + + "github.com/equinix-labs/otel-init-go/otelinit" + "github.com/go-logr/stdr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/dhcpv4/server4" + "github.com/tinkerbell/smee/backend/kube" + "github.com/tinkerbell/smee/dhcp" + "github.com/tinkerbell/smee/dhcp/handler" + "github.com/tinkerbell/smee/dhcp/handler/reservation" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" +) + +func main() { + ctx, done := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGHUP, syscall.SIGTERM) + defer done() + ctx, otelShutdown := otelinit.InitOpenTelemetry(ctx, "github.com/tinkerbell/smee/dhcp") + defer otelShutdown(ctx) + + l := stdr.New(log.New(os.Stdout, "", log.Lshortfile)) + l = l.WithName("github.com/tinkerbell/smee/dhcp") + // 1. create the backend + // 2. create the handler(backend) + // 3. create the listener(handler) + backend, err := kubeBackend(ctx) + if err != nil { + panic(err) + } + + h := &reservation.Handler{ + Log: l, + IPAddr: netip.MustParseAddr("192.168.2.50"), + Netboot: reservation.Netboot{ + IPXEBinServerTFTP: netip.MustParseAddrPort("192.168.2.50:69"), + IPXEBinServerHTTP: &url.URL{Scheme: "http", Host: "192.168.2.50:8080"}, + IPXEScriptURL: func(*dhcpv4.DHCPv4) *url.URL { + return &url.URL{Scheme: "http", Host: "192.168.2.50", Path: "auto.ipxe"} + }, + Enabled: true, + }, + OTELEnabled: true, + Backend: backend, + } + conn, err := server4.NewIPv4UDPConn("", net.UDPAddrFromAddrPort(netip.MustParseAddrPort("0.0.0.0:67"))) + if err != nil { + panic(err) + } + defer func() { + _ = conn.Close() + }() + server := &dhcp.Server{Logger: l, Conn: conn, Handlers: []dhcp.Handler{h}} + l.Info("starting server", "addr", h.IPAddr) + l.Error(server.Serve(ctx), "done") + l.Info("done") +} + +func kubeBackend(ctx context.Context) (handler.BackendReader, error) { + ccfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig( + &clientcmd.ClientConfigLoadingRules{ + ExplicitPath: "/home/tink/.kube/config", + }, + &clientcmd.ConfigOverrides{ + Context: api.Context{ + Namespace: "tink-system", + }, + }, + ) + + config, err := ccfg.ClientConfig() + if err != nil { + return nil, err + } + + k, err := kube.NewBackend(config) + if err != nil { + return nil, err + } + + go func() { + _ = k.Start(ctx) + }() + + return k, nil +} diff --git a/dhcp/handler/handler.go b/dhcp/handler/handler.go new file mode 100644 index 00000000..71dc6f46 --- /dev/null +++ b/dhcp/handler/handler.go @@ -0,0 +1,19 @@ +// Package handler holds the interface that backends implement, handlers take in, and the top level dhcp package passes to handlers. +package handler + +import ( + "context" + "net" + + "github.com/tinkerbell/smee/dhcp/data" +) + +// BackendReader is the interface for getting data from a backend. +// +// Backends implement this interface to provide DHCP and Netboot data to the handlers. +type BackendReader interface { + // Read data (from a backend) based on a mac address + // and return DHCP headers and options, including netboot info. + GetByMac(context.Context, net.HardwareAddr) (*data.DHCP, *data.Netboot, error) + GetByIP(context.Context, net.IP) (*data.DHCP, *data.Netboot, error) +} diff --git a/dhcp/handler/noop/noop.go b/dhcp/handler/noop/noop.go new file mode 100644 index 00000000..bae30661 --- /dev/null +++ b/dhcp/handler/noop/noop.go @@ -0,0 +1,28 @@ +// Package noop is a handler that does nothing. +package noop + +import ( + "context" + "log" + "net" + "os" + + "github.com/go-logr/logr" + "github.com/go-logr/stdr" + "github.com/tinkerbell/smee/dhcp/data" +) + +// Handler is a noop handler. +type Handler struct { + Log logr.Logger +} + +// Handle is the noop handler function. +func (n *Handler) Handle(_ context.Context, _ net.PacketConn, _ data.Packet) { + msg := "no handler specified. please specify a handler" + if n.Log.GetSink() == nil { + stdr.New(log.New(os.Stdout, "", log.Lshortfile)).Info(msg) + } else { + n.Log.Info(msg) + } +} diff --git a/dhcp/handler/noop/noop_test.go b/dhcp/handler/noop/noop_test.go new file mode 100644 index 00000000..8f0d5854 --- /dev/null +++ b/dhcp/handler/noop/noop_test.go @@ -0,0 +1,45 @@ +package noop + +import ( + "bytes" + "context" + "io" + "os" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/tinkerbell/smee/dhcp/data" + "github.com/tonglil/buflogr" +) + +func TestNoop_Handle(t *testing.T) { + var buf bytes.Buffer + n := &Handler{ + Log: buflogr.NewWithBuffer(&buf), + } + n.Handle(context.TODO(), nil, data.Packet{}) + want := "INFO no handler specified. please specify a handler\n" + if diff := cmp.Diff(buf.String(), want); diff != "" { + t.Fatalf(diff) + } +} + +func TestNoop_HandleSTDOUT(t *testing.T) { + old := os.Stdout + r, w, _ := os.Pipe() + os.Stdout = w + + n := &Handler{} + n.Handle(context.TODO(), nil, data.Packet{}) + + w.Close() + os.Stdout = old + + var buf bytes.Buffer + io.Copy(&buf, r) + + want := `noop.go:24: "level"=0 "msg"="no handler specified. please specify a handler"` + "\n" + if diff := cmp.Diff(buf.String(), want); diff != "" { + t.Fatalf(diff) + } +} diff --git a/dhcp/handler/reservation/handler.go b/dhcp/handler/reservation/handler.go new file mode 100644 index 00000000..ebe7461b --- /dev/null +++ b/dhcp/handler/reservation/handler.go @@ -0,0 +1,268 @@ +package reservation + +import ( + "context" + "errors" + "fmt" + "net" + "strings" + + "github.com/go-logr/logr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/tinkerbell/smee/backend/noop" + "github.com/tinkerbell/smee/dhcp/data" + oteldhcp "github.com/tinkerbell/smee/dhcp/otel" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + "golang.org/x/net/ipv4" +) + +const tracerName = "github.com/tinkerbell/smee/dhcp/server" + +// setDefaults will update the Handler struct to have default values so as +// to avoid panic for nil pointers and such. +func (h *Handler) setDefaults() { + if h.Backend == nil { + h.Backend = noop.Handler{} + } + if h.Log.GetSink() == nil { + h.Log = logr.Discard() + } +} + +// Handle responds to DHCP messages with DHCP server options. +func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, p data.Packet) { + h.setDefaults() + if p.Pkt == nil { + h.Log.Error(errors.New("incoming packet is nil"), "not able to respond when the incoming packet is nil") + return + } + upeer, ok := p.Peer.(*net.UDPAddr) + if !ok { + h.Log.Error(errors.New("peer is not a UDP connection"), "not able to respond when the peer is not a UDP connection") + return + } + if upeer == nil { + h.Log.Error(errors.New("peer is nil"), "not able to respond when the peer is nil") + return + } + if conn == nil { + h.Log.Error(errors.New("connection is nil"), "not able to respond when the connection is nil") + return + } + + var ifName string + if p.Md != nil { + ifName = p.Md.IfName + } + log := h.Log.WithValues("mac", p.Pkt.ClientHWAddr.String(), "xid", p.Pkt.TransactionID.String(), "interface", ifName) + tracer := otel.Tracer(tracerName) + var span trace.Span + ctx, span = tracer.Start( + ctx, + fmt.Sprintf("DHCP Packet Received: %v", p.Pkt.MessageType().String()), + trace.WithAttributes(h.encodeToAttributes(p.Pkt, "request")...), + trace.WithAttributes(attribute.String("DHCP.peer", p.Peer.String())), + trace.WithAttributes(attribute.String("DHCP.server.ifname", ifName)), + ) + + defer span.End() + + var reply *dhcpv4.DHCPv4 + switch mt := p.Pkt.MessageType(); mt { + case dhcpv4.MessageTypeDiscover: + d, n, err := h.readBackend(ctx, p.Pkt.ClientHWAddr) + if err != nil { + if hardwareNotFound(err) { + span.SetStatus(codes.Ok, "no reservation found") + return + } + log.Info("error reading from backend", "error", err) + span.SetStatus(codes.Error, err.Error()) + + return + } + log.Info("received DHCP packet", "type", p.Pkt.MessageType().String()) + reply = h.updateMsg(ctx, p.Pkt, d, n, dhcpv4.MessageTypeOffer) + log = log.WithValues("type", dhcpv4.MessageTypeOffer.String()) + case dhcpv4.MessageTypeRequest: + d, n, err := h.readBackend(ctx, p.Pkt.ClientHWAddr) + if err != nil { + if hardwareNotFound(err) { + span.SetStatus(codes.Ok, "no reservation found") + return + } + log.Info("error reading from backend", "error", err) + span.SetStatus(codes.Error, err.Error()) + + return + } + log.Info("received DHCP packet", "type", p.Pkt.MessageType().String()) + reply = h.updateMsg(ctx, p.Pkt, d, n, dhcpv4.MessageTypeAck) + log = log.WithValues("type", dhcpv4.MessageTypeAck.String()) + case dhcpv4.MessageTypeRelease: + // Since the design of this DHCP server is that all IP addresses are + // Host reservations, when a client releases an address, the server + // doesn't have anything to do. This case is included for clarity of this + // design decision. + log.Info("received DHCP release packet, no response required, all IPs are host reservations", "type", p.Pkt.MessageType().String()) + span.SetStatus(codes.Ok, "received release, no response required") + + return + default: + log.Info("received unknown message type", "type", p.Pkt.MessageType().String()) + span.SetStatus(codes.Error, "received unknown message type") + + return + } + + if bf := reply.BootFileName; bf != "" { + log = log.WithValues("bootFileName", bf) + } + if ns := reply.ServerIPAddr; ns != nil { + log = log.WithValues("nextServer", ns.String()) + } + log = log.WithValues("ipAddress", reply.YourIPAddr.String(), "destination", p.Peer.String()) + cm := &ipv4.ControlMessage{} + if p.Md != nil { + cm.IfIndex = p.Md.IfIndex + } + if _, err := conn.WriteTo(reply.ToBytes(), cm, p.Peer); err != nil { + log.Error(err, "failed to send DHCP") + span.SetStatus(codes.Error, err.Error()) + + return + } + + log.Info("sent DHCP response") + span.SetAttributes(h.encodeToAttributes(reply, "reply")...) + span.SetStatus(codes.Ok, "sent DHCP response") +} + +// readBackend encapsulates the backend read and opentelemetry handling. +func (h *Handler) readBackend(ctx context.Context, mac net.HardwareAddr) (*data.DHCP, *data.Netboot, error) { + h.setDefaults() + + tracer := otel.Tracer(tracerName) + ctx, span := tracer.Start(ctx, "Hardware data get") + defer span.End() + + d, n, err := h.Backend.GetByMac(ctx, mac) + if err != nil { + span.SetStatus(codes.Error, err.Error()) + + return nil, nil, err + } + + span.SetAttributes(d.EncodeToAttributes()...) + span.SetAttributes(n.EncodeToAttributes()...) + span.SetStatus(codes.Ok, "done reading from backend") + + return d, n, nil +} + +// updateMsg handles updating DHCP packets with the data from the backend. +func (h *Handler) updateMsg(ctx context.Context, pkt *dhcpv4.DHCPv4, d *data.DHCP, n *data.Netboot, msgType dhcpv4.MessageType) *dhcpv4.DHCPv4 { + h.setDefaults() + mods := []dhcpv4.Modifier{ + dhcpv4.WithMessageType(msgType), + dhcpv4.WithGeneric(dhcpv4.OptionServerIdentifier, h.IPAddr.AsSlice()), + dhcpv4.WithServerIP(h.IPAddr.AsSlice()), + } + mods = append(mods, h.setDHCPOpts(ctx, pkt, d)...) + + if h.Netboot.Enabled && h.isNetbootClient(pkt) == nil { + mods = append(mods, h.setNetworkBootOpts(ctx, pkt, n)) + } + reply, err := dhcpv4.NewReplyFromRequest(pkt, mods...) + if err != nil { + return nil + } + + return reply +} + +// isNetbootClient returns true if the client is a valid netboot client. +// +// A valid netboot client will have the following in its DHCP request: +// 1. is a DHCP discovery/request message type. +// 2. option 93 is set. +// 3. option 94 is set. +// 4. option 97 is correct length. +// 5. option 60 is set with this format: "PXEClient:Arch:xxxxx:UNDI:yyyzzz" or "HTTPClient:Arch:xxxxx:UNDI:yyyzzz". +// +// See: http://www.pix.net/software/pxeboot/archive/pxespec.pdf +// +// See: https://www.rfc-editor.org/rfc/rfc4578.html +func (h *Handler) isNetbootClient(pkt *dhcpv4.DHCPv4) error { + h.setDefaults() + var err error + // only response to DISCOVER and REQUEST packets + if pkt.MessageType() != dhcpv4.MessageTypeDiscover && pkt.MessageType() != dhcpv4.MessageTypeRequest { + // h.Log.Info("not a netboot client", "reason", "message type must be either Discover or Request", "mac", pkt.ClientHWAddr.String(), "message type", pkt.MessageType()) + err = errors.New("message type must be either Discover or Request") + } + // option 60 must be set + if !pkt.Options.Has(dhcpv4.OptionClassIdentifier) { + // h.Log.Info("not a netboot client", "reason", "option 60 not set", "mac", pkt.ClientHWAddr.String()) + err = fmt.Errorf("%w: option 60 not set", err) + } + // option 60 must start with PXEClient or HTTPClient + opt60 := pkt.GetOneOption(dhcpv4.OptionClassIdentifier) + if !strings.HasPrefix(string(opt60), string(pxeClient)) && !strings.HasPrefix(string(opt60), string(httpClient)) { + // h.Log.Info("not a netboot client", "reason", "option 60 not PXEClient or HTTPClient", "mac", pkt.ClientHWAddr.String(), "option 60", string(opt60)) + err = fmt.Errorf("%w: option 60 not PXEClient or HTTPClient", err) + } + + // option 93 must be set + if !pkt.Options.Has(dhcpv4.OptionClientSystemArchitectureType) { + // h.Log.Info("not a netboot client", "reason", "option 93 not set", "mac", pkt.ClientHWAddr.String()) + err = fmt.Errorf("%w: option 93 not set", err) + } + + // option 94 must be set + if !pkt.Options.Has(dhcpv4.OptionClientNetworkInterfaceIdentifier) { + // h.Log.Info("not a netboot client", "reason", "option 94 not set", "mac", pkt.ClientHWAddr.String()) + err = fmt.Errorf("%w: option 94 not set", err) + } + + // option 97 must be have correct length or not be set + guid := pkt.GetOneOption(dhcpv4.OptionClientMachineIdentifier) + switch len(guid) { + case 0: + // A missing GUID is invalid according to the spec, however + // there are PXE ROMs in the wild that omit the GUID and still + // expect to boot. The only thing we do with the GUID is + // mirror it back to the client if it's there, so we might as + // well accept these buggy ROMs. + case 17: + if guid[0] != 0 { + h.Log.Info("not a netboot client", "reason", "option 97 does not start with 0", "mac", pkt.ClientHWAddr.String(), "option 97", string(guid)) + err = fmt.Errorf("%w: option 97 does not start with 0", err) + } + default: + h.Log.Info("not a netboot client", "reason", "option 97 has invalid length (0 or 17)", "mac", pkt.ClientHWAddr.String(), "option 97", string(guid)) + err = fmt.Errorf("%w: option 97 has invalid length (0 or 17)", err) + } + + return err +} + +// encodeToAttributes takes a DHCP packet and returns opentelemetry key/value attributes. +func (h *Handler) encodeToAttributes(d *dhcpv4.DHCPv4, namespace string) []attribute.KeyValue { + h.setDefaults() + a := &oteldhcp.Encoder{Log: h.Log} + + return a.Encode(d, namespace, oteldhcp.AllEncoders()...) +} + +// hardwareNotFound returns true if the error is from a hardware record not being found. +func hardwareNotFound(err error) bool { + type hardwareNotFound interface { + NotFound() bool + } + te, ok := err.(hardwareNotFound) + return ok && te.NotFound() +} diff --git a/dhcp/handler/reservation/handler_test.go b/dhcp/handler/reservation/handler_test.go new file mode 100644 index 00000000..9dc177d3 --- /dev/null +++ b/dhcp/handler/reservation/handler_test.go @@ -0,0 +1,603 @@ +package reservation + +import ( + "context" + "errors" + "fmt" + "log" + "net" + "net/netip" + "net/url" + "os" + "testing" + "time" + + "github.com/go-logr/logr" + "github.com/go-logr/stdr" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/iana" + "github.com/insomniacslk/dhcp/rfc1035label" + "github.com/tinkerbell/smee/dhcp/data" + "github.com/tinkerbell/smee/dhcp/otel" + "go.opentelemetry.io/otel/attribute" + "golang.org/x/net/ipv4" + "golang.org/x/net/nettest" +) + +var errBadBackend = fmt.Errorf("bad backend") + +type mockBackend struct { + err error + allowNetboot bool + ipxeScript *url.URL + hardwareNotFound bool +} + +type hwNotFoundError struct{} + +func (hwNotFoundError) NotFound() bool { return true } +func (hwNotFoundError) Error() string { return "not found" } + +func (m *mockBackend) GetByMac(context.Context, net.HardwareAddr) (*data.DHCP, *data.Netboot, error) { + if m.err != nil { + return nil, nil, m.err + } + if m.hardwareNotFound { + return nil, nil, hwNotFoundError{} + } + d := &data.DHCP{ + MACAddress: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + IPAddress: netip.MustParseAddr("192.168.1.100"), + SubnetMask: []byte{255, 255, 255, 0}, + DefaultGateway: netip.MustParseAddr("192.168.1.1"), + NameServers: []net.IP{ + {1, 1, 1, 1}, + }, + Hostname: "test-host", + DomainName: "mydomain.com", + BroadcastAddress: netip.MustParseAddr("192.168.1.255"), + NTPServers: []net.IP{ + {132, 163, 96, 2}, + }, + LeaseTime: 60, + DomainSearch: []string{ + "mydomain.com", + }, + } + n := &data.Netboot{ + AllowNetboot: m.allowNetboot, + IPXEScriptURL: m.ipxeScript, + } + + return d, n, m.err +} + +func (m *mockBackend) GetByIP(context.Context, net.IP) (*data.DHCP, *data.Netboot, error) { + if m.hardwareNotFound { + return nil, nil, hwNotFoundError{} + } + return nil, nil, errors.New("not implemented") +} + +func TestHandle(t *testing.T) { + tests := map[string]struct { + server Handler + req *dhcpv4.DHCPv4 + want *dhcpv4.DHCPv4 + wantErr error + nilPeer bool + }{ + "success discover message type with netboot options": { + server: Handler{ + Backend: &mockBackend{ + allowNetboot: true, + ipxeScript: &url.URL{Scheme: "http", Host: "localhost:8181", Path: "auto.ipxe"}, + }, + IPAddr: netip.MustParseAddr("127.0.0.1"), + Netboot: Netboot{Enabled: true}, + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptUserClass("Tinkerbell"), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_X86_64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05}), + ), + }, + want: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootReply, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + ClientIPAddr: []byte{0, 0, 0, 0}, + YourIPAddr: []byte{192, 168, 1, 100}, + ServerIPAddr: []byte{0, 0, 0, 0}, + GatewayIPAddr: []byte{0, 0, 0, 0}, + BootFileName: "http://localhost:8181/auto.ipxe", + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeOffer), + dhcpv4.OptServerIdentifier(net.IP{127, 0, 0, 1}), + dhcpv4.OptIPAddressLeaseTime(time.Minute), + dhcpv4.OptSubnetMask(net.IPMask(net.IP{255, 255, 255, 0}.To4())), + dhcpv4.OptRouter([]net.IP{{192, 168, 1, 1}}...), + dhcpv4.OptDNS([]net.IP{{1, 1, 1, 1}}...), + dhcpv4.OptDomainName("mydomain.com"), + dhcpv4.OptHostName("test-host"), + dhcpv4.OptBroadcastAddress(net.IP{192, 168, 1, 255}), + dhcpv4.OptNTPServers([]net.IP{{132, 163, 96, 2}}...), + dhcpv4.OptDomainSearch(&rfc1035label.Labels{Labels: []string{"mydomain.com"}}), + dhcpv4.OptClassIdentifier("HTTPClient"), + dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, dhcpv4.Options{ + 6: []byte{8}, + 69: otel.TraceparentFromContext(context.Background()), + }.ToBytes()), + ), + }, + }, + "failure discover message type": { + server: Handler{ + Backend: &mockBackend{err: errBadBackend}, + IPAddr: netip.MustParseAddr("127.0.0.1"), + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + ), + }, + wantErr: errBadBackend, + }, + "success request message type with netboot options": { + server: Handler{ + Backend: &mockBackend{ + allowNetboot: true, + ipxeScript: &url.URL{Scheme: "http", Host: "localhost:8181", Path: "auto.ipxe"}, + }, + Netboot: Netboot{Enabled: true}, + IPAddr: netip.MustParseAddr("127.0.0.1"), + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + ClientIPAddr: []byte{0, 0, 0, 0}, + YourIPAddr: []byte{192, 168, 1, 100}, + ServerIPAddr: []byte{127, 0, 0, 1}, + GatewayIPAddr: []byte{0, 0, 0, 0}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeRequest), + dhcpv4.OptServerIdentifier(net.IP{127, 0, 0, 1}), + dhcpv4.OptIPAddressLeaseTime(time.Minute), + dhcpv4.OptSubnetMask(net.IPMask(net.IP{255, 255, 255, 0}.To4())), + dhcpv4.OptRouter([]net.IP{{192, 168, 1, 1}}...), + dhcpv4.OptDNS([]net.IP{{1, 1, 1, 1}}...), + dhcpv4.OptDomainName("mydomain.com"), + dhcpv4.OptHostName("test-host"), + dhcpv4.OptBroadcastAddress(net.IP{192, 168, 1, 255}), + dhcpv4.OptNTPServers([]net.IP{{132, 163, 96, 2}}...), + dhcpv4.OptDomainSearch(&rfc1035label.Labels{Labels: []string{"mydomain.com"}}), + dhcpv4.OptUserClass("Tinkerbell"), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_X86_64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05}), + ), + }, + want: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootReply, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + ClientIPAddr: []byte{0, 0, 0, 0}, + YourIPAddr: []byte{192, 168, 1, 100}, + ServerIPAddr: []byte{0, 0, 0, 0}, + GatewayIPAddr: []byte{0, 0, 0, 0}, + BootFileName: "http://localhost:8181/auto.ipxe", + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeAck), + dhcpv4.OptServerIdentifier(net.IP{127, 0, 0, 1}), + dhcpv4.OptIPAddressLeaseTime(time.Minute), + dhcpv4.OptSubnetMask(net.IPMask(net.IP{255, 255, 255, 0}.To4())), + dhcpv4.OptRouter([]net.IP{{192, 168, 1, 1}}...), + dhcpv4.OptDNS([]net.IP{{1, 1, 1, 1}}...), + dhcpv4.OptDomainName("mydomain.com"), + dhcpv4.OptHostName("test-host"), + dhcpv4.OptBroadcastAddress(net.IP{192, 168, 1, 255}), + dhcpv4.OptNTPServers([]net.IP{{132, 163, 96, 2}}...), + dhcpv4.OptDomainSearch(&rfc1035label.Labels{Labels: []string{"mydomain.com"}}), + dhcpv4.OptClassIdentifier("HTTPClient"), + dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, dhcpv4.Options{ + 6: []byte{8}, + 69: otel.TraceparentFromContext(context.Background()), + }.ToBytes()), + ), + }, + }, + "failure request message type": { + server: Handler{ + Backend: &mockBackend{err: errBadBackend}, + IPAddr: netip.MustParseAddr("127.0.0.1"), + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeRequest), + ), + }, + wantErr: errBadBackend, + }, + "request release type": { + server: Handler{ + Backend: &mockBackend{err: errBadBackend}, + IPAddr: netip.MustParseAddr("127.0.0.1"), + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeRelease), + ), + }, + wantErr: errBadBackend, + }, + "unknown message type": { + server: Handler{ + Backend: &mockBackend{err: errBadBackend}, + IPAddr: netip.MustParseAddr("127.0.0.1"), + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeInform), + ), + }, + wantErr: errBadBackend, + }, + "fail WriteTo": { + server: Handler{ + Backend: &mockBackend{}, + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + ), + }, + wantErr: errBadBackend, + nilPeer: true, + }, + "nil incoming packet": { + want: nil, + wantErr: errBadBackend, + }, + /*"nil incoming packet": { + want: nil, + wantErr: errBadBackend, + },*/ + "failure no hardware found discover": { + server: Handler{ + Backend: &mockBackend{hardwareNotFound: true}, + IPAddr: netip.MustParseAddr("127.0.0.1"), + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + ), + }, + want: nil, + wantErr: errBadBackend, + }, + "failure no hardware found request": { + server: Handler{ + Backend: &mockBackend{hardwareNotFound: true}, + IPAddr: netip.MustParseAddr("127.0.0.1"), + }, + req: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeRequest), + ), + }, + want: nil, + wantErr: errBadBackend, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + s := tt.server + conn, err := nettest.NewLocalPacketListener("udp") + if err != nil { + t.Fatal("1", err) + } + defer conn.Close() + + pc, err := net.ListenPacket("udp4", ":0") + if err != nil { + t.Fatal("2", err) + } + defer pc.Close() + peer := &net.UDPAddr{IP: net.IP{127, 0, 0, 1}, Port: pc.LocalAddr().(*net.UDPAddr).Port} + if tt.nilPeer { + peer = nil + } + + con := ipv4.NewPacketConn(conn) + con.SetControlMessage(ipv4.FlagInterface, true) + + n, err := net.InterfaceByName("lo") + if err != nil { + t.Fatal(err) + } + s.Handle(context.Background(), con, data.Packet{Peer: peer, Pkt: tt.req, Md: &data.Metadata{IfName: n.Name, IfIndex: n.Index}}) + + msg, err := client(pc) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("client() error = %v, wantErr %v", err, tt.wantErr) + } + + if diff := cmp.Diff(msg, tt.want, cmpopts.IgnoreUnexported(dhcpv4.DHCPv4{})); diff != "" { + t.Fatal("diff", diff) + } + }) + } +} + +func client(pc net.PacketConn) (*dhcpv4.DHCPv4, error) { + buf := make([]byte, 1024) + pc.SetReadDeadline(time.Now().Add(time.Millisecond * 100)) + if _, _, err := pc.ReadFrom(buf); err != nil { + return nil, errBadBackend + } + msg, err := dhcpv4.FromBytes(buf) + if err != nil { + return nil, errBadBackend + } + + return msg, nil +} + +func TestUpdateMsg(t *testing.T) { + type args struct { + m *dhcpv4.DHCPv4 + data *data.DHCP + netboot *data.Netboot + msg dhcpv4.MessageType + } + tests := map[string]struct { + args args + want *dhcpv4.DHCPv4 + wantErr bool + }{ + "success": { + args: args{ + m: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptUserClass("Tinkerbell"), + dhcpv4.OptClassIdentifier("HTTPClient"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05}), + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + ), + }, + data: &data.DHCP{IPAddress: netip.MustParseAddr("192.168.1.100"), SubnetMask: net.IPMask(net.IP{255, 255, 255, 0}.To4())}, + netboot: &data.Netboot{AllowNetboot: true, IPXEScriptURL: &url.URL{Scheme: "http", Host: "localhost:8181", Path: "auto.ipxe"}}, + msg: dhcpv4.MessageTypeDiscover, + }, + want: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootReply, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + YourIPAddr: []byte{192, 168, 1, 100}, + ClientIPAddr: []byte{0, 0, 0, 0}, + BootFileName: "http://localhost:8181/auto.ipxe", + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptServerIdentifier(net.IP{127, 0, 0, 1}), + dhcpv4.OptIPAddressLeaseTime(3600), + dhcpv4.OptSubnetMask(net.IPMask(net.IP{255, 255, 255, 0}.To4())), + dhcpv4.OptClassIdentifier("HTTPClient"), + dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, dhcpv4.Options{ + 6: []byte{8}, + 69: otel.TraceparentFromContext(context.Background()), + }.ToBytes()), + ), + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + s := &Handler{ + Log: stdr.New(log.New(os.Stdout, "", log.Lshortfile)), + IPAddr: netip.MustParseAddr("127.0.0.1"), + Netboot: Netboot{ + Enabled: true, + }, + Backend: &mockBackend{ + allowNetboot: true, + ipxeScript: &url.URL{Scheme: "http", Host: "localhost:8181", Path: "auto.ipxe"}, + }, + // Listener: netip.AddrPortFrom(netip.MustParseAddr("127.0.0.1"), 67), + } + got := s.updateMsg(context.Background(), tt.args.m, tt.args.data, tt.args.netboot, tt.args.msg) + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(dhcpv4.DHCPv4{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestOne(t *testing.T) { + t.Skip() + h := &Handler{} + _, _, err := h.readBackend(context.Background(), nil) + t.Fatal(err) +} + +func TestReadBackend(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + wantDHCP *data.DHCP + wantNetboot *data.Netboot + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptUserClass("Tinkerbell"), + dhcpv4.OptClassIdentifier("HTTPClient"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05}), + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + ), + }, + wantDHCP: &data.DHCP{ + MACAddress: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + IPAddress: netip.MustParseAddr("192.168.1.100"), + SubnetMask: []byte{255, 255, 255, 0}, + DefaultGateway: netip.MustParseAddr("192.168.1.1"), + NameServers: []net.IP{{1, 1, 1, 1}}, + Hostname: "test-host", + DomainName: "mydomain.com", + BroadcastAddress: netip.MustParseAddr("192.168.1.255"), + NTPServers: []net.IP{{132, 163, 96, 2}}, + LeaseTime: 60, + DomainSearch: []string{"mydomain.com"}, + }, + wantNetboot: &data.Netboot{AllowNetboot: true, IPXEScriptURL: &url.URL{Scheme: "http", Host: "localhost:8181", Path: "auto.ipxe"}}, + wantErr: nil, + }, + "failure": { + input: &dhcpv4.DHCPv4{}, + wantErr: errBadBackend, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + s := &Handler{ + Log: stdr.New(log.New(os.Stdout, "", log.Lshortfile)), + IPAddr: netip.MustParseAddr("127.0.0.1"), + Netboot: Netboot{ + Enabled: true, + }, + Backend: &mockBackend{ + err: tt.wantErr, + allowNetboot: true, + ipxeScript: &url.URL{Scheme: "http", Host: "localhost:8181", Path: "auto.ipxe"}, + }, + // Listener: netip.AddrPortFrom(netip.MustParseAddr("127.0.0.1"), 67), + } + netaddrComparer := cmp.Comparer(func(x, y netip.Addr) bool { + i := x.Compare(y) + return i == 0 + }) + gotDHCP, gotNetboot, err := s.readBackend(context.Background(), tt.input.ClientHWAddr) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("gotErr: %v, wantErr: %v", err, tt.wantErr) + } + if diff := cmp.Diff(gotDHCP, tt.wantDHCP, netaddrComparer); diff != "" { + t.Fatal(diff) + } + if diff := cmp.Diff(gotNetboot, tt.wantNetboot); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestIsNetbootClient(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want error + }{ + "fail invalid message type": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptMessageType(dhcpv4.MessageTypeInform))}, want: errors.New("")}, + "fail no opt60": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover))}, want: errors.New("")}, + "fail bad opt60": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("BadClient"), + )}, want: errors.New("")}, + "fail no opt93": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + )}, want: errors.New("")}, + "fail no opt94": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + )}, want: errors.New("")}, + "fail invalid opt97[0] != 0": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05}), + )}, want: errors.New("")}, + "fail invalid len(opt97)": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x01, 0x02}), + )}, want: errors.New("")}, + "success len(opt97) == 0": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{}), + )}, want: nil}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + s := &Handler{Log: logr.Discard()} + if err := s.isNetbootClient(tt.input); (err == nil) != (tt.want == nil) { + t.Errorf("isNetbootClient() = %v, want %v", err, tt.want) + } + }) + } +} + +func TestEncodeToAttributes(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want []attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{BootFileName: "snp.efi"}, + want: []attribute.KeyValue{ + attribute.String("DHCP.testing.Header.file", "snp.efi"), + attribute.String("DHCP.testing.Header.flags", "Unicast"), + attribute.String("DHCP.testing.Header.transactionID", "0x00000000"), + }, + }, + "error": {}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + stdr.SetVerbosity(1) + s := &Handler{Log: stdr.New(log.New(os.Stdout, "", log.Lshortfile))} + kvs := s.encodeToAttributes(tt.input, "testing") + got := attribute.NewSet(kvs...) + want := attribute.NewSet(tt.want...) + enc := attribute.DefaultEncoder() + if diff := cmp.Diff(got.Encoded(enc), want.Encoded(enc)); diff != "" { + t.Log(got.Encoded(enc)) + t.Log(want.Encoded(enc)) + t.Fatal(diff) + } + }) + } +} diff --git a/dhcp/handler/reservation/option.go b/dhcp/handler/reservation/option.go new file mode 100644 index 00000000..0d3d8121 --- /dev/null +++ b/dhcp/handler/reservation/option.go @@ -0,0 +1,226 @@ +package reservation + +import ( + "context" + "fmt" + "net" + "net/netip" + "net/url" + "strings" + + "github.com/equinix-labs/otel-init-go/otelhelpers" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/iana" + "github.com/tinkerbell/smee/dhcp/data" + "github.com/tinkerbell/smee/dhcp/otel" +) + +// UserClass is DHCP option 77 (https://www.rfc-editor.org/rfc/rfc3004.html). +type UserClass string + +// clientType is from DHCP option 60. Normally only PXEClient or HTTPClient. +type clientType string + +const ( + pxeClient clientType = "PXEClient" + httpClient clientType = "HTTPClient" +) + +// known user-class types. must correspond to DHCP option 77 - User-Class +// https://www.rfc-editor.org/rfc/rfc3004.html +const ( + // If the client has had iPXE burned into its ROM (or is a VM + // that uses iPXE as the PXE "ROM"), special handling is + // needed because in this mode the client is using iPXE native + // drivers and chainloading to a UNDI stack won't work. + IPXE UserClass = "iPXE" + // If the client identifies as "Tinkerbell", we've already + // chainloaded this client to the full-featured copy of iPXE + // we supply. We have to distinguish this case so we don't + // loop on the chainload step. + Tinkerbell UserClass = "Tinkerbell" +) + +// ArchToBootFile maps supported hardware PXE architectures types to iPXE binary files. +var ArchToBootFile = map[iana.Arch]string{ + iana.INTEL_X86PC: "undionly.kpxe", + iana.NEC_PC98: "undionly.kpxe", + iana.EFI_ITANIUM: "undionly.kpxe", + iana.DEC_ALPHA: "undionly.kpxe", + iana.ARC_X86: "undionly.kpxe", + iana.INTEL_LEAN_CLIENT: "undionly.kpxe", + iana.EFI_IA32: "ipxe.efi", + iana.EFI_X86_64: "ipxe.efi", + iana.EFI_XSCALE: "ipxe.efi", + iana.EFI_BC: "ipxe.efi", + iana.EFI_ARM32: "snp.efi", + iana.EFI_ARM64: "snp.efi", + iana.EFI_X86_HTTP: "ipxe.efi", + iana.EFI_X86_64_HTTP: "ipxe.efi", + iana.EFI_ARM32_HTTP: "snp.efi", + iana.EFI_ARM64_HTTP: "snp.efi", + iana.Arch(41): "snp.efi", // arm rpiboot: https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture +} + +// String function for clientType. +func (c clientType) String() string { + return string(c) +} + +// String function for UserClass. +func (u UserClass) String() string { + return string(u) +} + +// setDHCPOpts takes a client dhcp packet and data (typically from a backend) and creates a slice of DHCP packet modifiers. +// m is the DHCP request from a client. d is the data to use to create the DHCP packet modifiers. +// This is most likely the place where we would have any business logic for determining DHCP option setting. +func (h *Handler) setDHCPOpts(_ context.Context, _ *dhcpv4.DHCPv4, d *data.DHCP) []dhcpv4.Modifier { + mods := []dhcpv4.Modifier{ + dhcpv4.WithLeaseTime(d.LeaseTime), + dhcpv4.WithYourIP(d.IPAddress.AsSlice()), + } + if len(d.NameServers) > 0 { + mods = append(mods, dhcpv4.WithDNS(d.NameServers...)) + } + if len(d.DomainSearch) > 0 { + mods = append(mods, dhcpv4.WithDomainSearchList(d.DomainSearch...)) + } + if len(d.NTPServers) > 0 { + mods = append(mods, dhcpv4.WithOption(dhcpv4.OptNTPServers(d.NTPServers...))) + } + if d.BroadcastAddress.Compare(netip.Addr{}) != 0 { + mods = append(mods, dhcpv4.WithGeneric(dhcpv4.OptionBroadcastAddress, d.BroadcastAddress.AsSlice())) + } + if d.DomainName != "" { + mods = append(mods, dhcpv4.WithGeneric(dhcpv4.OptionDomainName, []byte(d.DomainName))) + } + if d.Hostname != "" { + mods = append(mods, dhcpv4.WithGeneric(dhcpv4.OptionHostName, []byte(d.Hostname))) + } + if len(d.SubnetMask) > 0 { + mods = append(mods, dhcpv4.WithNetmask(d.SubnetMask)) + } + if d.DefaultGateway.Compare(netip.Addr{}) != 0 { + mods = append(mods, dhcpv4.WithRouter(d.DefaultGateway.AsSlice())) + } + if h.SyslogAddr.Compare(netip.Addr{}) != 0 { + mods = append(mods, dhcpv4.WithOption(dhcpv4.OptGeneric(dhcpv4.OptionLogServer, h.SyslogAddr.AsSlice()))) + } + + return mods +} + +// setNetworkBootOpts purpose is to sets 3 or 4 values. 2 DHCP headers, option 43 and optionally option (60). +// These headers and options are returned as a dhcvp4.Modifier that can be used to modify a dhcp response. +// github.com/insomniacslk/dhcp uses this method to simplify packet manipulation. +// +// DHCP Headers (https://datatracker.ietf.org/doc/html/rfc2131#section-2) +// 'siaddr': IP address of next bootstrap server. represented below as `.ServerIPAddr`. +// 'file': Client boot file name. represented below as `.BootFileName`. +// +// DHCP option +// option 60: Class Identifier. https://www.rfc-editor.org/rfc/rfc2132.html#section-9.13 +// option 60 is set if the client's option 60 (Class Identifier) starts with HTTPClient. +func (h *Handler) setNetworkBootOpts(ctx context.Context, m *dhcpv4.DHCPv4, n *data.Netboot) dhcpv4.Modifier { + // m is a received DHCPv4 packet. + // d is the reply packet we are building. + withNetboot := func(d *dhcpv4.DHCPv4) { + var opt60 string + // if the client sends opt 60 with HTTPClient then we need to respond with opt 60 + if val := m.Options.Get(dhcpv4.OptionClassIdentifier); val != nil { + if strings.HasPrefix(string(val), httpClient.String()) { + d.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClassIdentifier, []byte(httpClient))) + opt60 = httpClient.String() + } + } + d.BootFileName = "/netboot-not-allowed" + d.ServerIPAddr = net.IPv4(0, 0, 0, 0) + if n.AllowNetboot { + a := arch(m) + bin, found := ArchToBootFile[a] + if !found { + h.Log.Error(fmt.Errorf("unable to find bootfile for arch"), "network boot not allowed", "arch", a, "archInt", int(a), "mac", m.ClientHWAddr) + return + } + uClass := UserClass(string(m.GetOneOption(dhcpv4.OptionUserClassInformation))) + var ipxeScript *url.URL + if h.Netboot.IPXEScriptURL != nil { + ipxeScript = h.Netboot.IPXEScriptURL(m) + } + if n.IPXEScriptURL != nil { + ipxeScript = n.IPXEScriptURL + } + d.BootFileName, d.ServerIPAddr = h.bootfileAndNextServer(ctx, uClass, opt60, bin, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, ipxeScript) + pxe := dhcpv4.Options{ // FYI, these are suboptions of option43. ref: https://datatracker.ietf.org/doc/html/rfc2132#section-8.4 + // PXE Boot Server Discovery Control - bypass, just boot from filename. + 6: []byte{8}, + 69: otel.TraceparentFromContext(ctx), + } + d.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, pxe.ToBytes())) + } + } + + return withNetboot +} + +// bootfileAndNextServer returns the bootfile (string) and next server (net.IP). +// input arguments `tftp`, `ipxe` and `iscript` use non string types so as to attempt to be more clear about the expectation around what is wanted for these values. +// It also helps us avoid having to validate a string in multiple ways. +func (h *Handler) bootfileAndNextServer(ctx context.Context, uClass UserClass, opt60, bin string, tftp netip.AddrPort, ipxe, iscript *url.URL) (string, net.IP) { + var nextServer net.IP + var bootfile string + if tp := otelhelpers.TraceparentStringFromContext(ctx); h.OTELEnabled && tp != "" { + bin = fmt.Sprintf("%s-%v", bin, tp) + } + // If a machine is in an ipxe boot loop, it is likely to be that we aren't matching on IPXE or Tinkerbell userclass (option 77). + switch { // order matters here. + case uClass == Tinkerbell, (h.Netboot.UserClass != "" && uClass == h.Netboot.UserClass): // this case gets us out of an ipxe boot loop. + bootfile = "/no-ipxe-script-defined" + if iscript != nil { + bootfile = iscript.String() + } + case clientType(opt60) == httpClient: // Check the client type from option 60. + bootfile = ipxe.JoinPath(bin).String() + nextServer = net.ParseIP("0.0.0.0") + if n, err := netip.ParseAddrPort(ipxe.Host); err == nil { + nextServer = n.Addr().AsSlice() + } else if n2 := net.ParseIP(ipxe.Host); n2 != nil { + nextServer = net.ParseIP(ipxe.Host) + } + case uClass == IPXE: // if the "iPXE" user class is found it means we aren't in our custom version of ipxe, but because of the option 43 we're setting we need to give a full tftp url from which to boot. + bootfile = fmt.Sprintf("tftp://%v/%v", tftp.String(), bin) + nextServer = net.IP(tftp.Addr().AsSlice()) + default: + bootfile = bin + nextServer = net.IP(tftp.Addr().AsSlice()) + } + + return bootfile, nextServer +} + +// arch returns the arch of the client pulled from DHCP option 93. +func arch(d *dhcpv4.DHCPv4) iana.Arch { + // get option 93 ; arch + fwt := d.ClientArch() + if len(fwt) == 0 { + return iana.Arch(255) // unknown arch + } + var archKnown bool + var a iana.Arch + for _, elem := range fwt { + if !strings.Contains(elem.String(), "unknown") { + archKnown = true + // Basic architecture identification, based purely on + // the PXE architecture option. + // https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture + a = elem + break + } + } + if !archKnown { + return iana.Arch(255) // unknown arch + } + + return a +} diff --git a/dhcp/handler/reservation/option_test.go b/dhcp/handler/reservation/option_test.go new file mode 100644 index 00000000..0e3add44 --- /dev/null +++ b/dhcp/handler/reservation/option_test.go @@ -0,0 +1,337 @@ +package reservation + +import ( + "context" + "net" + "net/netip" + "net/url" + "testing" + "time" + + "github.com/equinix-labs/otel-init-go/otelhelpers" + "github.com/go-logr/logr" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/iana" + "github.com/insomniacslk/dhcp/rfc1035label" + "github.com/tinkerbell/smee/dhcp/data" + oteldhcp "github.com/tinkerbell/smee/dhcp/otel" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" +) + +func TestSetDHCPOpts(t *testing.T) { + type args struct { + in0 context.Context + m *dhcpv4.DHCPv4 + d *data.DHCP + } + tests := map[string]struct { + server Handler + args args + want *dhcpv4.DHCPv4 + }{ + "success": { + server: Handler{Log: logr.Discard(), SyslogAddr: netip.MustParseAddr("192.168.7.7")}, + args: args{ + in0: context.Background(), + m: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptParameterRequestList(dhcpv4.OptionSubnetMask))}, + d: &data.DHCP{ + MACAddress: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + IPAddress: netip.MustParseAddr("192.168.4.4"), + SubnetMask: []byte{255, 255, 255, 0}, + DefaultGateway: netip.MustParseAddr("192.168.4.1"), + NameServers: []net.IP{ + {8, 8, 8, 8}, + {8, 8, 4, 4}, + }, + Hostname: "test-server", + DomainName: "mynet.local", + BroadcastAddress: netip.MustParseAddr("192.168.4.255"), + NTPServers: []net.IP{ + {132, 163, 96, 2}, + {132, 163, 96, 3}, + }, + LeaseTime: 84600, + DomainSearch: []string{ + "mynet.local", + }, + }, + }, + want: &dhcpv4.DHCPv4{ + OpCode: dhcpv4.OpcodeBootRequest, + HWType: iana.HWTypeEthernet, + ClientHWAddr: net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}, + ClientIPAddr: []byte{0, 0, 0, 0}, + YourIPAddr: []byte{192, 168, 4, 4}, + ServerIPAddr: []byte{0, 0, 0, 0}, + GatewayIPAddr: []byte{0, 0, 0, 0}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptGeneric(dhcpv4.OptionLogServer, []byte{192, 168, 7, 7}), + dhcpv4.OptSubnetMask(net.IPMask{255, 255, 255, 0}), + dhcpv4.OptBroadcastAddress(net.IP{192, 168, 4, 255}), + dhcpv4.OptIPAddressLeaseTime(time.Duration(84600)*time.Second), + dhcpv4.OptDomainName("mynet.local"), + dhcpv4.OptHostName("test-server"), + dhcpv4.OptRouter(net.IP{192, 168, 4, 1}), + dhcpv4.OptDNS([]net.IP{ + {8, 8, 8, 8}, + {8, 8, 4, 4}, + }...), + dhcpv4.OptNTPServers([]net.IP{ + {132, 163, 96, 2}, + {132, 163, 96, 3}, + }...), + dhcpv4.OptDomainSearch(&rfc1035label.Labels{ + Labels: []string{"mynet.local"}, + }), + ), + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + s := &Handler{ + Log: tt.server.Log, + Netboot: Netboot{ + IPXEBinServerTFTP: tt.server.Netboot.IPXEBinServerTFTP, + IPXEBinServerHTTP: tt.server.Netboot.IPXEBinServerHTTP, + IPXEScriptURL: tt.server.Netboot.IPXEScriptURL, + Enabled: tt.server.Netboot.Enabled, + UserClass: tt.server.Netboot.UserClass, + }, + IPAddr: tt.server.IPAddr, + Backend: tt.server.Backend, + SyslogAddr: tt.server.SyslogAddr, + } + mods := s.setDHCPOpts(tt.args.in0, tt.args.m, tt.args.d) + finalPkt, err := dhcpv4.New(mods...) + if err != nil { + t.Fatalf("setDHCPOpts() error = %v, wantErr nil", err) + } + if diff := cmp.Diff(tt.want, finalPkt, cmpopts.IgnoreFields(dhcpv4.DHCPv4{}, "TransactionID")); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestArch(t *testing.T) { + tests := map[string]struct { + pkt *dhcpv4.DHCPv4 + want iana.Arch + }{ + "found": { + pkt: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptClientArch(iana.INTEL_X86PC))}, + want: iana.INTEL_X86PC, + }, + "unknown": { + pkt: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptClientArch(iana.Arch(255)))}, + want: iana.Arch(255), + }, + "unknown: opt 93 len 0": { + pkt: &dhcpv4.DHCPv4{}, + want: iana.Arch(255), + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := arch(tt.pkt) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestBootfileAndNextServer(t *testing.T) { + type args struct { + mac net.HardwareAddr + uClass UserClass + opt60 string + bin string + tftp netip.AddrPort + ipxe *url.URL + iscript *url.URL + } + tests := map[string]struct { + server *Handler + args args + otelEnabled bool + wantBootFile string + wantNextSrv net.IP + }{ + "success bootfile only": { + server: &Handler{Log: logr.Discard()}, + args: args{ + uClass: Tinkerbell, + iscript: &url.URL{Scheme: "http", Host: "localhost:8080", Path: "/auto.ipxe"}, + }, + wantBootFile: "http://localhost:8080/auto.ipxe", + wantNextSrv: nil, + }, + "success httpClient": { + server: &Handler{Log: logr.Discard()}, + args: args{ + mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + opt60: httpClient.String(), + bin: "snp.ipxe", + ipxe: &url.URL{Scheme: "http", Host: "localhost:8181"}, + }, + wantBootFile: "http://localhost:8181/snp.ipxe", + wantNextSrv: net.IPv4(0, 0, 0, 0), + }, + "success userclass iPXE": { + server: &Handler{Log: logr.Discard()}, + args: args{ + mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + uClass: IPXE, + bin: "unidonly.kpxe", + tftp: netip.MustParseAddrPort("192.168.6.5:69"), + ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, + }, + wantBootFile: "tftp://192.168.6.5:69/unidonly.kpxe", + wantNextSrv: net.ParseIP("192.168.6.5"), + }, + "success userclass iPXE with otel": { + server: &Handler{Log: logr.Discard(), OTELEnabled: true}, + otelEnabled: true, + args: args{ + mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + uClass: IPXE, + bin: "unidonly.kpxe", + tftp: netip.MustParseAddrPort("192.168.6.5:69"), + ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, + }, + wantBootFile: "tftp://192.168.6.5:69/unidonly.kpxe-00-23b1e307bb35484f535a1f772c06910e-d887dc3912240434-01", + wantNextSrv: net.ParseIP("192.168.6.5"), + }, + "success default": { + server: &Handler{Log: logr.Discard()}, + args: args{ + mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + bin: "unidonly.kpxe", + tftp: netip.MustParseAddrPort("192.168.6.5:69"), + ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, + }, + wantBootFile: "unidonly.kpxe", + wantNextSrv: net.ParseIP("192.168.6.5"), + }, + "success otel enabled, no traceparent": { + server: &Handler{Log: logr.Discard(), OTELEnabled: true}, + args: args{ + mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + bin: "unidonly.kpxe", + tftp: netip.MustParseAddrPort("192.168.6.5:69"), + ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, + }, + wantBootFile: "unidonly.kpxe", + wantNextSrv: net.ParseIP("192.168.6.5"), + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + ctx := context.Background() + if tt.otelEnabled { + // set global propagator to tracecontext (the default is no-op). + prop := propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}) + otel.SetTextMapPropagator(prop) + ctx = otelhelpers.ContextWithTraceparentString(ctx, "00-23b1e307bb35484f535a1f772c06910e-d887dc3912240434-01") + } + bootfile, nextServer := tt.server.bootfileAndNextServer(ctx, tt.args.uClass, tt.args.opt60, tt.args.bin, tt.args.tftp, tt.args.ipxe, tt.args.iscript) + if diff := cmp.Diff(tt.wantBootFile, bootfile); diff != "" { + t.Fatal("bootfile", diff) + } + if diff := cmp.Diff(tt.wantNextSrv, nextServer); diff != "" { + t.Fatal("nextServer", diff) + } + }) + } +} + +func TestSetNetworkBootOpts(t *testing.T) { + type args struct { + in0 context.Context + m *dhcpv4.DHCPv4 + n *data.Netboot + } + tests := map[string]struct { + server *Handler + args args + want *dhcpv4.DHCPv4 + }{ + "netboot not allowed": { + server: &Handler{Log: logr.Discard()}, + args: args{ + in0: context.Background(), + m: &dhcpv4.DHCPv4{}, + n: &data.Netboot{AllowNetboot: false}, + }, + want: &dhcpv4.DHCPv4{ServerIPAddr: net.IPv4(0, 0, 0, 0), BootFileName: "/netboot-not-allowed"}, + }, + "netboot allowed": { + server: &Handler{Log: logr.Discard(), Netboot: Netboot{IPXEScriptURL: func(*dhcpv4.DHCPv4) *url.URL { + return &url.URL{Scheme: "http", Host: "localhost:8181", Path: "/01:02:03:04:05:06/auto.ipxe"} + }}}, + args: args{ + in0: context.Background(), + m: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptUserClass(Tinkerbell.String()), + dhcpv4.OptClassIdentifier("HTTPClient:xxxxx"), + dhcpv4.OptClientArch(iana.EFI_X86_64_HTTP), + ), + }, + n: &data.Netboot{AllowNetboot: true, IPXEScriptURL: &url.URL{Scheme: "http", Host: "localhost:8181", Path: "/01:02:03:04:05:06/auto.ipxe"}}, + }, + want: &dhcpv4.DHCPv4{BootFileName: "http://localhost:8181/01:02:03:04:05:06/auto.ipxe", Options: dhcpv4.OptionsFromList( + dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, dhcpv4.Options{ + 6: []byte{8}, + 69: oteldhcp.TraceparentFromContext(context.Background()), + }.ToBytes()), + dhcpv4.OptClassIdentifier("HTTPClient"), + )}, + }, + "netboot not allowed, arch unknown": { + server: &Handler{Log: logr.Discard(), Netboot: Netboot{IPXEScriptURL: func(*dhcpv4.DHCPv4) *url.URL { + return &url.URL{Scheme: "http", Host: "localhost:8181", Path: "/01:02:03:04:05:06/auto.ipxe"} + }}}, + args: args{ + in0: context.Background(), + m: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptUserClass(Tinkerbell.String()), + dhcpv4.OptClientArch(iana.UBOOT_ARM64), + ), + }, + n: &data.Netboot{AllowNetboot: true}, + }, + want: &dhcpv4.DHCPv4{ServerIPAddr: net.IPv4(0, 0, 0, 0), BootFileName: "/netboot-not-allowed"}, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + s := &Handler{ + Log: tt.server.Log, + Netboot: Netboot{ + IPXEBinServerTFTP: tt.server.Netboot.IPXEBinServerTFTP, + IPXEBinServerHTTP: tt.server.Netboot.IPXEBinServerHTTP, + IPXEScriptURL: tt.server.Netboot.IPXEScriptURL, + Enabled: tt.server.Netboot.Enabled, + UserClass: tt.server.Netboot.UserClass, + }, + IPAddr: tt.server.IPAddr, + Backend: tt.server.Backend, + } + gotFunc := s.setNetworkBootOpts(tt.args.in0, tt.args.m, tt.args.n) + got := new(dhcpv4.DHCPv4) + gotFunc(got) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/dhcp/handler/reservation/reservation.go b/dhcp/handler/reservation/reservation.go new file mode 100644 index 00000000..a0d60e7e --- /dev/null +++ b/dhcp/handler/reservation/reservation.go @@ -0,0 +1,56 @@ +// Package reservation is the handler for responding to DHCPv4 messages with only host reservations. +package reservation + +import ( + "net/netip" + "net/url" + + "github.com/go-logr/logr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/tinkerbell/smee/dhcp/handler" +) + +// Handler holds the configuration details for the running the DHCP server. +type Handler struct { + // Backend is the backend to use for getting DHCP data. + Backend handler.BackendReader + + // IPAddr is the IP address to use in DHCP responses. + // Option 54 and the sname DHCP header. + // This could be a load balancer IP address or an ingress IP address or a local IP address. + IPAddr netip.Addr + + // Log is used to log messages. + // `logr.Discard()` can be used if no logging is desired. + Log logr.Logger + + // Netboot configuration + Netboot Netboot + + // OTELEnabled is used to determine if netboot options include otel naming. + // When true, the netboot filename will be appended with otel information. + // For example, the filename will be "snp.efi-00-23b1e307bb35484f535a1f772c06910e-d887dc3912240434-01". + // -00--- + OTELEnabled bool + + // SyslogAddr is the address to send syslog messages to. DHCP Option 7. + SyslogAddr netip.Addr +} + +// Netboot holds the netboot configuration details used in running a DHCP server. +type Netboot struct { + // iPXE binary server IP:Port serving via TFTP. + IPXEBinServerTFTP netip.AddrPort + + // IPXEBinServerHTTP is the URL to the IPXE binary server serving via HTTP(s). + IPXEBinServerHTTP *url.URL + + // IPXEScriptURL is the URL to the IPXE script to use. + IPXEScriptURL func(*dhcpv4.DHCPv4) *url.URL + + // Enabled is whether to enable sending netboot DHCP options. + Enabled bool + + // UserClass (for network booting) allows a custom DHCP option 77 to be used to break out of an iPXE loop. + UserClass UserClass +} diff --git a/dhcp/otel/otel.go b/dhcp/otel/otel.go new file mode 100644 index 00000000..aca8275c --- /dev/null +++ b/dhcp/otel/otel.go @@ -0,0 +1,369 @@ +// Package otel handles translating DHCP headers and options to otel key/value attributes. +package otel + +import ( + "context" + "fmt" + "net" + "strings" + + "github.com/go-logr/logr" + "github.com/insomniacslk/dhcp/dhcpv4" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +const keyNamespace = "DHCP" + +// Encoder holds the otel key/value attributes. +type Encoder struct { + Log logr.Logger +} + +type notFoundError struct { + optName string +} + +func (e *notFoundError) Error() string { + return fmt.Sprintf("%q not found in DHCP packet", e.optName) +} + +func (e *notFoundError) found() bool { + return true +} + +type found interface { + found() bool +} + +// OptNotFound returns true if err is an option not found error. +func OptNotFound(err error) bool { + te, ok := err.(found) + return ok && te.found() +} + +// Encode runs a slice of encoders against a DHCPv4 packet turning the values into opentelemetry attribute key/value pairs. +func (e *Encoder) Encode(pkt *dhcpv4.DHCPv4, namespace string, encoders ...func(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error)) []attribute.KeyValue { + if e.Log.GetSink() == nil { + e.Log = logr.Discard() + } + var attrs []attribute.KeyValue + for _, elem := range encoders { + kv, err := elem(pkt, namespace) + if err != nil { + e.Log.V(2).Info("opentelemetry attribute not added", "error", fmt.Sprintf("%v", err)) + continue + } + attrs = append(attrs, kv) + } + + return attrs +} + +// AllEncoders returns a slice of all available DHCP otel encoders. +func AllEncoders() []func(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + return []func(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error){ + EncodeFlags, EncodeTransactionID, + EncodeYIADDR, EncodeSIADDR, + EncodeCHADDR, EncodeFILE, + EncodeOpt1, EncodeOpt3, EncodeOpt6, + EncodeOpt12, EncodeOpt15, EncodeOpt28, + EncodeOpt42, EncodeOpt51, EncodeOpt53, + EncodeOpt54, EncodeOpt60, EncodeOpt93, + EncodeOpt94, EncodeOpt97, EncodeOpt119, + } +} + +// EncodeFlags takes DHCP flags from a DHCP packet and returns an OTEL key/value pair. +// key/value pair. See https://datatracker.ietf.org/doc/html/rfc2131#page-9 +func EncodeFlags(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Header.flags", keyNamespace, namespace) + if d != nil { + return attribute.String(key, d.FlagsToString()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeTransactionID takes the Transaction ID header from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeTransactionID(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Header.transactionID", keyNamespace, namespace) + if d != nil { + return attribute.String(key, d.TransactionID.String()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt1 takes DHCP Opt 1 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt1(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + opt := "Opt1.SubnetMask" + key := fmt.Sprintf("%v.%v.%v", keyNamespace, namespace, opt) + if d != nil && d.SubnetMask() != nil { + sm := net.IP(d.SubnetMask()).String() + return attribute.String(key, sm), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: opt} +} + +// EncodeOpt3 takes DHCP Opt 3 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt3(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt3.DefaultGateway", keyNamespace, namespace) + if d != nil { + var routers []string + for _, e := range d.Router() { + routers = append(routers, e.String()) + } + if len(routers) > 0 { + return attribute.String(key, strings.Join(routers, ",")), nil + } + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt6 takes DHCP Opt 6 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt6(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt6.NameServers", keyNamespace, namespace) + if d != nil { + var ns []string + for _, e := range d.DNS() { + ns = append(ns, e.String()) + } + if len(ns) > 0 { + return attribute.String(key, strings.Join(ns, ",")), nil + } + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt12 takes DHCP Opt 12 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt12(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt12.Hostname", keyNamespace, namespace) + if d != nil && d.HostName() != "" { + return attribute.String(key, d.HostName()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt15 takes DHCP Opt 15 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt15(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt15.DomainName", keyNamespace, namespace) + if d != nil && d.DomainName() != "" { + return attribute.String(key, d.DomainName()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt28 takes DHCP Opt 28 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt28(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt28.BroadcastAddress", keyNamespace, namespace) + if d != nil && d.BroadcastAddress() != nil { + return attribute.String(key, d.BroadcastAddress().String()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt42 takes DHCP Opt 42 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt42(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt42.NTPServers", keyNamespace, namespace) + if d != nil { + var ntp []string + for _, e := range d.NTPServers() { + ntp = append(ntp, e.String()) + } + if len(ntp) > 0 { + return attribute.String(key, strings.Join(ntp, ",")), nil + } + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt51 takes DHCP Opt 51 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt51(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt51.LeaseTime", keyNamespace, namespace) + if d != nil && d.IPAddressLeaseTime(0) != 0 { + return attribute.Float64(key, d.IPAddressLeaseTime(0).Seconds()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt53 takes DHCP Opt 53 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt53(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt53.MessageType", keyNamespace, namespace) + if d != nil && d.MessageType() != dhcpv4.MessageTypeNone { + return attribute.String(key, d.MessageType().String()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt54 takes DHCP Opt 54 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt54(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt54.ServerIdentifier", keyNamespace, namespace) + if d != nil && d.ServerIdentifier() != nil { + return attribute.String(key, d.ServerIdentifier().String()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt60 takes DHCP Opt 60 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt60(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt60.ClassIdentifier", keyNamespace, namespace) + if d != nil && d.ClassIdentifier() != "" { + return attribute.String(key, d.ClassIdentifier()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt93 takes DHCP Opt 93 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt93(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt93.ClientIdentifier", keyNamespace, namespace) + if d != nil && len(d.ClientArch()) > 0 { + var r []string + for _, i := range d.ClientArch() { + r = append(r, i.String()) + } + + return attribute.StringSlice(key, r), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt94 takes DHCP Opt 94 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt94(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt94.ClientNetworkInterfaceIdentifier", keyNamespace, namespace) + if d != nil && len(d.GetOneOption(dhcpv4.OptionClientNetworkInterfaceIdentifier)) > 0 { + var r []string + for _, i := range d.GetOneOption(dhcpv4.OptionClientNetworkInterfaceIdentifier) { + r = append(r, fmt.Sprintf("%v", i)) + } + + // "." delimited follows the same format from tcpdump + return attribute.String(key, strings.Join(r, ".")), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt97 takes DHCP Opt 97 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt97(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt97.ClientMachineIdentifier", keyNamespace, namespace) + if d != nil && len(d.GetOneOption(dhcpv4.OptionClientMachineIdentifier)) > 0 { + var r []string + for _, i := range d.GetOneOption(dhcpv4.OptionClientMachineIdentifier) { + r = append(r, fmt.Sprintf("%v", i)) + } + + // "." delimited follows the same format from tcpdump + return attribute.String(key, strings.Join(r, ".")), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeOpt119 takes DHCP Opt 119 from a DHCP packet and returns an OTEL key/value pair. +// See https://www.iana.org/assignments/bootp-dhcp-parameters/bootp-dhcp-parameters.xhtml +func EncodeOpt119(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Opt119.DomainSearch", keyNamespace, namespace) + if d != nil { + if l := d.DomainSearch(); l != nil { + return attribute.String(key, strings.Join(l.Labels, ",")), nil + } + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeYIADDR takes the yiaddr header from a DHCP packet and returns an OTEL +// key/value pair. See https://datatracker.ietf.org/doc/html/rfc2131#page-9 +func EncodeYIADDR(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Header.yiaddr", keyNamespace, namespace) + if d != nil && d.YourIPAddr != nil { + return attribute.String(key, d.YourIPAddr.String()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeSIADDR takes the siaddr header from a DHCP packet and returns an OTEL +// key/value pair. See https://datatracker.ietf.org/doc/html/rfc2131#page-9 +func EncodeSIADDR(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Header.siaddr", keyNamespace, namespace) + if d != nil && d.ServerIPAddr != nil { + return attribute.String(key, d.ServerIPAddr.String()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeCHADDR takes the CHADDR header from a DHCP packet and returns an OTEL +// key/value pair. See https://datatracker.ietf.org/doc/html/rfc2131#page-9 +func EncodeCHADDR(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Header.chaddr", keyNamespace, namespace) + if d != nil && d.ClientHWAddr != nil { + return attribute.String(key, d.ClientHWAddr.String()), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// EncodeFILE takes the file header from a DHCP packet and returns an OTEL +// key/value pair. See https://datatracker.ietf.org/doc/html/rfc2131#page-9 +func EncodeFILE(d *dhcpv4.DHCPv4, namespace string) (attribute.KeyValue, error) { + key := fmt.Sprintf("%v.%v.Header.file", keyNamespace, namespace) + if d != nil && d.BootFileName != "" { + return attribute.String(key, d.BootFileName), nil + } + + return attribute.KeyValue{}, ¬FoundError{optName: key} +} + +// TraceparentFromContext extracts the binary trace id, span id, and trace flags +// from the running span in ctx and returns a 26 byte []byte with the traceparent +// encoded and ready to pass into a suboption (most likely 69) of opt43. +func TraceparentFromContext(ctx context.Context) []byte { + sc := trace.SpanContextFromContext(ctx) + tpBytes := make([]byte, 0, 26) + + // the otel spec says 16 bytes for trace id and 8 for spans are good enough + // for everyone copy them into a []byte that we can deliver over option43 + tid := [16]byte(sc.TraceID()) // type TraceID [16]byte + sid := [8]byte(sc.SpanID()) // type SpanID [8]byte + + tpBytes = append(tpBytes, 0x00) // traceparent version + tpBytes = append(tpBytes, tid[:]...) // trace id + tpBytes = append(tpBytes, sid[:]...) // span id + if sc.IsSampled() { + tpBytes = append(tpBytes, 0x01) // trace flags + } else { + tpBytes = append(tpBytes, 0x00) + } + + return tpBytes +} diff --git a/dhcp/otel/otel_test.go b/dhcp/otel/otel_test.go new file mode 100644 index 00000000..cc1da5a6 --- /dev/null +++ b/dhcp/otel/otel_test.go @@ -0,0 +1,636 @@ +package otel + +import ( + "bytes" + "context" + "net" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/iana" + "github.com/insomniacslk/dhcp/rfc1035label" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" +) + +func TestEncode(t *testing.T) { + tests := map[string]struct { + allEncoders bool + pkt *dhcpv4.DHCPv4 + want []attribute.KeyValue + }{ + "no encoders": {pkt: &dhcpv4.DHCPv4{}, want: nil}, + "all encoders": {allEncoders: true, pkt: &dhcpv4.DHCPv4{BootFileName: "ipxe.efi", Flags: 0}, want: []attribute.KeyValue{ + {Key: attribute.Key("DHCP.test.Header.flags"), Value: attribute.StringValue("Unicast")}, + {Key: attribute.Key("DHCP.test.Header.transactionID"), Value: attribute.StringValue("0x00000000")}, + {Key: attribute.Key("DHCP.test.Header.file"), Value: attribute.StringValue("ipxe.efi")}, + }}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + e := &Encoder{} + got := e.Encode(tt.pkt, "test") + if tt.allEncoders { + got = e.Encode(tt.pkt, "test", AllEncoders()...) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Logf("%+v", got) + t.Fatal(diff) + } + }) + } +} + +func TestEncodeError(t *testing.T) { + tests := map[string]struct { + input *notFoundError + want string + }{ + "success": {input: ¬FoundError{optName: "opt1"}, want: "\"opt1\" not found in DHCP packet"}, + "success nil error": {input: ¬FoundError{}, want: "\"\" not found in DHCP packet"}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := tt.input.Error() + if diff := cmp.Diff(got, tt.want); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt1(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptSubnetMask(net.IPMask(net.IP{255, 255, 255, 0}.To4())), + )}, + want: attribute.String("DHCP.testing.Opt1.SubnetMask", "255.255.255.0"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt1(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt1() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt3(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptRouter([]net.IP{{192, 168, 1, 1}}...), + )}, + want: attribute.String("DHCP.testing.Opt3.DefaultGateway", "192.168.1.1"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt3(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt13() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt6(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptDNS([]net.IP{{1, 1, 1, 1}}...), + )}, + want: attribute.String("DHCP.testing.Opt6.NameServers", "1.1.1.1"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt6(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt6() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt12(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptHostName("test-host"), + )}, + want: attribute.String("DHCP.testing.Opt12.Hostname", "test-host"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt12(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt12() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt15(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptDomainName("example.com"), + )}, + want: attribute.String("DHCP.testing.Opt15.DomainName", "example.com"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt15(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt15() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt28(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptBroadcastAddress(net.IP{192, 168, 1, 255}), + )}, + want: attribute.String("DHCP.testing.Opt28.BroadcastAddress", "192.168.1.255"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt28(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt28() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt42(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptNTPServers([]net.IP{{132, 163, 96, 2}}...), + )}, + want: attribute.String("DHCP.testing.Opt42.NTPServers", "132.163.96.2"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt42(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt42() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt51(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptIPAddressLeaseTime(time.Minute), + )}, + want: attribute.String("DHCP.testing.Opt51.LeaseTime", "60"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt51(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt51() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt53(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeOffer), + )}, + want: attribute.String("DHCP.testing.Opt53.MessageType", "OFFER"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt53(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt53() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt54(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptServerIdentifier(net.IP{127, 0, 0, 1}), + )}, + want: attribute.String("DHCP.testing.Opt54.ServerIdentifier", "127.0.0.1"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt54(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt54() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt60(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptClassIdentifier("foobar"), + )}, + want: attribute.String("DHCP.testing.Opt60.ClassIdentifier", "foobar"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt60(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt60() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt93(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptClientArch(iana.INTEL_X86PC), + )}, + want: attribute.StringSlice("DHCP.testing.Opt93.ClientIdentifier", []string{"Intel x86PC"}), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt93(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt93() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Log(tt.input.ClientArch()) + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt94(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x01}), + )}, + want: attribute.String("DHCP.testing.Opt94.ClientNetworkInterfaceIdentifier", "1.2.1"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt94(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt94() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Log(tt.input.ClientArch()) + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt97(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f, 0x10}), + )}, + want: attribute.String("DHCP.testing.Opt97.ClientMachineIdentifier", "0.1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt97(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt97() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Log(tt.input.GetOneOption(dhcpv4.OptionClientMachineIdentifier)) + t.Fatal(diff) + } + }) + } +} + +func TestSetOpt119(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptDomainSearch(&rfc1035label.Labels{Labels: []string{"mydomain.com"}}), + )}, + want: attribute.String("DHCP.testing.Opt119.DomainSearch", "mydomain.com"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeOpt119(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setOpt119() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetHeaderFlags(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{}, + want: attribute.String("DHCP.testing.Header.flags", "Unicast"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeFlags(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setHeaderFlags() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetHeaderTransactionID(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{TransactionID: dhcpv4.TransactionID{0x00, 0x00, 0x00, 0x00}}, + want: attribute.String("DHCP.testing.Header.transactionID", "0x00000000"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeTransactionID(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("EncodeTransactionID() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetHeaderYIADDR(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{YourIPAddr: []byte{192, 168, 2, 100}}, + want: attribute.String("DHCP.testing.Header.yiaddr", "192.168.2.100"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeYIADDR(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setHeaderYIADDR() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetHeaderSIADDR(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{ServerIPAddr: []byte{127, 0, 0, 1}}, + want: attribute.String("DHCP.testing.Header.siaddr", "127.0.0.1"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeSIADDR(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setHeaderSIADDR() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetHeaderCHADDR(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}}, + want: attribute.String("DHCP.testing.Header.chaddr", "01:02:03:04:05:06"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeCHADDR(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setHeaderCHADDR() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestSetHeaderFILE(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want attribute.KeyValue + wantErr error + }{ + "success": { + input: &dhcpv4.DHCPv4{BootFileName: "snp.efi"}, + want: attribute.String("DHCP.testing.Header.file", "snp.efi"), + }, + "error": {wantErr: ¬FoundError{}}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got, err := EncodeFILE(tt.input, "testing") + if tt.wantErr != nil && !OptNotFound(err) { + t.Fatalf("setHeaderFILE() error (type: %T) = %[1]v, wantErr (type: %T) %[2]v", err, tt.wantErr) + } + if diff := cmp.Diff(got, tt.want, cmpopts.IgnoreUnexported(attribute.Value{})); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestTraceparentFromContext(t *testing.T) { + want := []byte{0, 1, 2, 3, 4, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 6, 7, 8, 0, 0, 0, 0, 1} + sc := trace.NewSpanContext(trace.SpanContextConfig{ + TraceID: trace.TraceID{0x01, 0x02, 0x03, 0x04}, + SpanID: trace.SpanID{0x05, 0x06, 0x07, 0x08}, + TraceFlags: trace.TraceFlags(1), + }) + rmSpan := trace.ContextWithRemoteSpanContext(context.Background(), sc) + + got := TraceparentFromContext(rmSpan) + if !bytes.Equal(got, want) { + t.Errorf("binaryTpFromContext() = %v, want %v", got, want) + } +} diff --git a/go.sum b/go.sum index 76308df0..d183573d 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/tinkerbell/ipxedust v0.0.0-20231215220341-a535c5deb47a h1:BNbuuQp8m/L github.com/tinkerbell/ipxedust v0.0.0-20231215220341-a535c5deb47a/go.mod h1:zrFXKJHUplvuggD9MzSQuZldQZU4CLter7QYqSLiiE4= github.com/tinkerbell/tink v0.9.0 h1:W7X/OEmhyYXE/kPVu1U31fpugVHoc2qsAvBtsZ7mkDg= github.com/tinkerbell/tink v0.9.0/go.mod h1:r8gDvx/Y+GEFeT9xwKa14ULrkMre8mYmH3/E9VbUkEw= +github.com/tonglil/buflogr v1.0.1 h1:WXFZLKxLfqcVSmckwiMCF8jJwjIgmStJmg63YKRF1p0= +github.com/tonglil/buflogr v1.0.1/go.mod h1:yYWwvSpn/3uAaqjf6mJg/XMiAciaR0QcRJH2gJGDxNE= github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 h1:YcojQL98T/OO+rybuzn2+5KrD5dBwXIvYBvQ2cD3Avg= github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= From 6ca7560bdb5a1ae48e9e3e0ed7e7b7a51dfeda0e Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Fri, 22 Dec 2023 21:08:45 -0700 Subject: [PATCH 02/17] Add proxyDHCP handler: This allows Smee to not have any IPAM responsibilities, especially during the network boot process. This is still a WIP and will be clean up in subsequent commits. Signed-off-by: Jacob Weinstock --- cmd/smee/main.go | 25 +- internal/dhcp/dhcp.go | 230 ++++++++++------ internal/dhcp/handler/proxy/proxy.go | 345 ++++++++++++++++++++++++ internal/dhcp/server/dhcp.go | 112 ++++++++ internal/dhcp/{ => server}/dhcp_test.go | 2 +- 5 files changed, 620 insertions(+), 94 deletions(-) create mode 100644 internal/dhcp/handler/proxy/proxy.go create mode 100644 internal/dhcp/server/dhcp.go rename internal/dhcp/{ => server}/dhcp_test.go (99%) diff --git a/cmd/smee/main.go b/cmd/smee/main.go index 594ad33f..8c1c31ed 100644 --- a/cmd/smee/main.go +++ b/cmd/smee/main.go @@ -22,9 +22,9 @@ import ( "github.com/insomniacslk/dhcp/dhcpv4/server4" "github.com/tinkerbell/ipxedust" "github.com/tinkerbell/ipxedust/ihttp" - "github.com/tinkerbell/smee/internal/dhcp" "github.com/tinkerbell/smee/internal/dhcp/handler" - "github.com/tinkerbell/smee/internal/dhcp/handler/reservation" + "github.com/tinkerbell/smee/internal/dhcp/handler/proxy" + "github.com/tinkerbell/smee/internal/dhcp/server" "github.com/tinkerbell/smee/internal/ipxe/http" "github.com/tinkerbell/smee/internal/ipxe/script" "github.com/tinkerbell/smee/internal/metric" @@ -239,7 +239,7 @@ func main() { panic(err) } defer conn.Close() - ds := &dhcp.Server{Logger: log, Conn: conn, Handlers: []dhcp.Handler{dh}} + ds := &server.DHCP{Logger: log, Conn: conn, Handlers: []server.Handler{dh}} return ds.Serve(ctx) }) @@ -252,7 +252,8 @@ func main() { log.Info("smee is shutting down") } -func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation.Handler, error) { +// func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation.Handler, error) { +func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*proxy.Handler, error) { // 1. create the handler // 2. create the backend // 3. add the backend to the handler @@ -287,7 +288,20 @@ func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation if err != nil { return nil, fmt.Errorf("invalid syslog address: %w", err) } - dh := &reservation.Handler{ + log.V(19).Info("debug", "syslog", syslogIP) + dh := &proxy.Handler{ + Backend: nil, + IPAddr: pktIP, + Log: log, + Netboot: proxy.Netboot{ + IPXEBinServerTFTP: tftpIP, + IPXEBinServerHTTP: httpBinaryURL, + IPXEScriptURL: ipxeScript, + Enabled: true, + }, + OTELEnabled: true, + } + /*dh := &reservation.Handler{ Backend: nil, IPAddr: pktIP, Log: log, @@ -300,6 +314,7 @@ func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation OTELEnabled: true, SyslogAddr: syslogIP, } + */ switch { case c.backends.file.Enabled && c.backends.kubernetes.Enabled: panic("only one backend can be enabled at a time") diff --git a/internal/dhcp/dhcp.go b/internal/dhcp/dhcp.go index 82c60c21..1f1630fa 100644 --- a/internal/dhcp/dhcp.go +++ b/internal/dhcp/dhcp.go @@ -1,112 +1,166 @@ -// Package dhcp providers UDP listening and serving functionality. package dhcp import ( - "context" - "net" + "fmt" + "strings" - "github.com/go-logr/logr" "github.com/insomniacslk/dhcp/dhcpv4" - "github.com/insomniacslk/dhcp/dhcpv4/server4" - "github.com/tinkerbell/smee/internal/dhcp/data" - "golang.org/x/net/ipv4" + "github.com/insomniacslk/dhcp/iana" ) -// Handler is a type that defines the handler function to be called every time a -// valid DHCPv4 message is received -// type Handler func(ctx context.Context, conn net.PacketConn, d data.Packet). -type Handler interface { - Handle(ctx context.Context, conn *ipv4.PacketConn, d data.Packet) -} - -// Server represents a DHCPv4 server object. -type Server struct { - Conn net.PacketConn - Handlers []Handler - Logger logr.Logger -} +const ( + PXEClient ClientType = "PXEClient" + HTTPClient ClientType = "HTTPClient" +) -// Serve serves requests. -func (s *Server) Serve(ctx context.Context) error { - go func() { - <-ctx.Done() - _ = s.Close() - }() - s.Logger.Info("Server listening on", "addr", s.Conn.LocalAddr()) - - nConn := ipv4.NewPacketConn(s.Conn) - if err := nConn.SetControlMessage(ipv4.FlagInterface, true); err != nil { - s.Logger.Info("error setting control message", "err", err) - return err - } +// known user-class types. must correspond to DHCP option 77 - User-Class +// https://www.rfc-editor.org/rfc/rfc3004.html +const ( + // If the client has had iPXE burned into its ROM (or is a VM + // that uses iPXE as the PXE "ROM"), special handling is + // needed because in this mode the client is using iPXE native + // drivers and chainloading to a UNDI stack won't work. + IPXE UserClass = "iPXE" + // If the client identifies as "Tinkerbell", we've already + // chainloaded this client to the full-featured copy of iPXE + // we supply. We have to distinguish this case so we don't + // loop on the chainload step. + Tinkerbell UserClass = "Tinkerbell" +) - defer func() { - _ = nConn.Close() - }() - for { - // Max UDP packet size is 65535. Max DHCPv4 packet size is 576. An ethernet frame is 1500 bytes. - // We use 4096 as a reasonable buffer size. dhcpv4.FromBytes will handle the rest. - rbuf := make([]byte, 4096) - n, cm, peer, err := nConn.ReadFrom(rbuf) - if err != nil { - select { - case <-ctx.Done(): - return nil - default: - } - s.Logger.Info("error reading from packet conn", "err", err) - return err - } +// UserClass is DHCP option 77 (https://www.rfc-editor.org/rfc/rfc3004.html). +type UserClass string - m, err := dhcpv4.FromBytes(rbuf[:n]) - if err != nil { - s.Logger.Info("error parsing DHCPv4 request", "err", err) - continue - } +// ClientType is from DHCP option 60. Normally only PXEClient or HTTPClient. +type ClientType string - upeer, ok := peer.(*net.UDPAddr) - if !ok { - s.Logger.Info("not a UDP connection? Peer is", "peer", peer) - continue - } - // Set peer to broadcast if the client did not have an IP. - if upeer.IP == nil || upeer.IP.To4().Equal(net.IPv4zero) { - upeer = &net.UDPAddr{ - IP: net.IPv4bcast, - Port: upeer.Port, - } - } +// ArchToBootFile maps supported hardware PXE architectures types to iPXE binary files. +var ArchToBootFile = map[iana.Arch]string{ + iana.INTEL_X86PC: "undionly.kpxe", + iana.NEC_PC98: "undionly.kpxe", + iana.EFI_ITANIUM: "undionly.kpxe", + iana.DEC_ALPHA: "undionly.kpxe", + iana.ARC_X86: "undionly.kpxe", + iana.INTEL_LEAN_CLIENT: "undionly.kpxe", + iana.EFI_IA32: "ipxe.efi", + iana.EFI_X86_64: "ipxe.efi", + iana.EFI_XSCALE: "ipxe.efi", + iana.EFI_BC: "ipxe.efi", + iana.EFI_ARM32: "snp.efi", + iana.EFI_ARM64: "snp.efi", + iana.EFI_X86_HTTP: "ipxe.efi", + iana.EFI_X86_64_HTTP: "ipxe.efi", + iana.EFI_ARM32_HTTP: "snp.efi", + iana.EFI_ARM64_HTTP: "snp.efi", + iana.Arch(41): "snp.efi", // arm rpiboot: https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture +} - var ifName string - if n, err := net.InterfaceByIndex(cm.IfIndex); err == nil { - ifName = n.Name - } +// ErrUnknownArch is used when the PXE client request is from an unknown architecture. +var ErrUnknownArch = fmt.Errorf("could not determine client architecture from option 93") - for _, handler := range s.Handlers { - go handler.Handle(ctx, nConn, data.Packet{Peer: upeer, Pkt: m, Md: &data.Metadata{IfName: ifName, IfIndex: cm.IfIndex}}) +// Arch returns the Arch of the client pulled from DHCP option 93. +func Arch(d *dhcpv4.DHCPv4) iana.Arch { + // get option 93 ; arch + fwt := d.ClientArch() + if len(fwt) == 0 { + return iana.Arch(255) // unknown arch + } + var archKnown bool + var a iana.Arch + for _, elem := range fwt { + if !strings.Contains(elem.String(), "unknown") { + archKnown = true + // Basic architecture identification, based purely on + // the PXE architecture option. + // https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture + a = elem + break } } + if !archKnown { + return iana.Arch(255) // unknown arch + } + + return a +} + +// String function for clientType. +func (c ClientType) String() string { + return string(c) } -// Close sends a termination request to the server, and closes the UDP listener. -func (s *Server) Close() error { - return s.Conn.Close() +// String function for UserClass. +func (u UserClass) String() string { + return string(u) } -// NewServer initializes and returns a new Server object. -func NewServer(ifname string, addr *net.UDPAddr, handler ...Handler) (*Server, error) { - s := &Server{ - Handlers: handler, - Logger: logr.Discard(), +// IsNetbootClient returns true if the client is a valid netboot client. +// +// A valid netboot client will have the following in its DHCP request: +// 1. is a DHCP discovery/request message type. +// 2. option 93 is set. +// 3. option 94 is set. +// 4. option 97 is correct length. +// 5. option 60 is set with this format: "PXEClient:Arch:xxxxx:UNDI:yyyzzz" or "HTTPClient:Arch:xxxxx:UNDI:yyyzzz". +// +// See: http://www.pix.net/software/pxeboot/archive/pxespec.pdf +// +// See: https://www.rfc-editor.org/rfc/rfc4578.html +func IsNetbootClient(pkt *dhcpv4.DHCPv4) error { + var err error + // only response to DISCOVER and REQUEST packets + if pkt.MessageType() != dhcpv4.MessageTypeDiscover && pkt.MessageType() != dhcpv4.MessageTypeRequest { + // h.Log.Info("not a netboot client", "reason", "message type must be either Discover or Request", "mac", pkt.ClientHWAddr.String(), "message type", pkt.MessageType()) + err = wrapNonNil(err, "message type must be either Discover or Request") + } + // option 60 must be set + if !pkt.Options.Has(dhcpv4.OptionClassIdentifier) { + // h.Log.Info("not a netboot client", "reason", "option 60 not set", "mac", pkt.ClientHWAddr.String()) + err = wrapNonNil(err, "option 60 not set") + } + // option 60 must start with PXEClient or HTTPClient + opt60 := pkt.GetOneOption(dhcpv4.OptionClassIdentifier) + if !strings.HasPrefix(string(opt60), string(PXEClient)) && !strings.HasPrefix(string(opt60), string(HTTPClient)) { + // h.Log.Info("not a netboot client", "reason", "option 60 not PXEClient or HTTPClient", "mac", pkt.ClientHWAddr.String(), "option 60", string(opt60)) + err = wrapNonNil(err, "option 60 not PXEClient or HTTPClient") + } + + // option 93 must be set + if !pkt.Options.Has(dhcpv4.OptionClientSystemArchitectureType) { + // h.Log.Info("not a netboot client", "reason", "option 93 not set", "mac", pkt.ClientHWAddr.String()) + err = wrapNonNil(err, "option 93 not set") } - if s.Conn == nil { - var err error - conn, err := server4.NewIPv4UDPConn(ifname, addr) - if err != nil { - return nil, err + // option 94 must be set + if !pkt.Options.Has(dhcpv4.OptionClientNetworkInterfaceIdentifier) { + // h.Log.Info("not a netboot client", "reason", "option 94 not set", "mac", pkt.ClientHWAddr.String()) + err = wrapNonNil(err, "option 94 not set") + } + + // option 97 must be have correct length or not be set + guid := pkt.GetOneOption(dhcpv4.OptionClientMachineIdentifier) + switch len(guid) { + case 0: + // A missing GUID is invalid according to the spec, however + // there are PXE ROMs in the wild that omit the GUID and still + // expect to boot. The only thing we do with the GUID is + // mirror it back to the client if it's there, so we might as + // well accept these buggy ROMs. + case 17: + if guid[0] != 0 { + err = wrapNonNil(err, "option 97 does not start with 0") } - s.Conn = conn + default: + err = wrapNonNil(err, "option 97 has invalid length (must be 0 or 17)") } - return s, nil + + return err +} + +func wrapNonNil(err error, format string, a ...any) error { + if err == nil { + return fmt.Errorf(format, a...) + } + + return fmt.Errorf("%w: %w", err, fmt.Errorf(format, a...)) } diff --git a/internal/dhcp/handler/proxy/proxy.go b/internal/dhcp/handler/proxy/proxy.go new file mode 100644 index 00000000..36ddafb8 --- /dev/null +++ b/internal/dhcp/handler/proxy/proxy.go @@ -0,0 +1,345 @@ +/* + Package proxy implements a DHCP handler that provides proxyDHCP functionality. + +"[A] Proxy DHCP server behaves much like a DHCP server by listening for ordinary +DHCP client traffic and responding to certain client requests. However, unlike the +DHCP server, the PXE Proxy DHCP server does not administer network addresses, and +it only responds to clients that identify themselves as PXE clients. The responses +given by the PXE Proxy DHCP server contain the mechanism by which the client locates +the boot servers or the network addresses and descriptions of the supported, +compatible boot servers." + +Reference: https://www.ibm.com/docs/en/aix/7.1?topic=protocol-preboot-execution-environment-proxy-dhcp-daemon +*/ +package proxy + +import ( + "context" + "encoding/hex" + "fmt" + "net" + "net/netip" + "net/url" + "path/filepath" + "strings" + + "github.com/go-logr/logr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/iana" + "github.com/tinkerbell/smee/internal/dhcp" + "github.com/tinkerbell/smee/internal/dhcp/data" + "github.com/tinkerbell/smee/internal/dhcp/handler" + "golang.org/x/net/ipv4" +) + +// Handler holds the configuration details for the running the DHCP server. +type Handler struct { + // Backend is the backend to use for getting DHCP data. + Backend handler.BackendReader + + // IPAddr is the IP address to use in DHCP responses. + // Option 54 and the sname DHCP header. + // This could be a load balancer IP address or an ingress IP address or a local IP address. + IPAddr netip.Addr + + // Log is used to log messages. + // `logr.Discard()` can be used if no logging is desired. + Log logr.Logger + + // Netboot configuration + Netboot Netboot + + // OTELEnabled is used to determine if netboot options include otel naming. + // When true, the netboot filename will be appended with otel information. + // For example, the filename will be "snp.efi-00-23b1e307bb35484f535a1f772c06910e-d887dc3912240434-01". + // -00--- + OTELEnabled bool +} + +// Netboot holds the netboot configuration details used in running a DHCP server. +type Netboot struct { + // iPXE binary server IP:Port serving via TFTP. + IPXEBinServerTFTP netip.AddrPort + + // IPXEBinServerHTTP is the URL to the IPXE binary server serving via HTTP(s). + IPXEBinServerHTTP *url.URL + + // IPXEScriptURL is the URL to the IPXE script to use. + IPXEScriptURL func(*dhcpv4.DHCPv4) *url.URL + + // Enabled is whether to enable sending netboot DHCP options. + Enabled bool + + // UserClass (for network booting) allows a custom DHCP option 77 to be used to break out of an iPXE loop. + UserClass dhcp.UserClass +} + +// netbootClient describes a device that is requesting a network boot. +type netbootClient struct { + mac net.HardwareAddr + arch iana.Arch + uClass dhcp.UserClass + cType dhcp.ClientType +} + +// Redirection name comes from section 2.5 of http://www.pix.net/software/pxeboot/archive/pxespec.pdf +func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, data data.Packet) { + log := h.Log.WithValues("hwaddr", data.Pkt.ClientHWAddr.String(), "listenAddr", conn.LocalAddr()) + reply, err := dhcpv4.New(dhcpv4.WithReply(data.Pkt), + dhcpv4.WithGatewayIP(data.Pkt.GatewayIPAddr), + dhcpv4.WithOptionCopied(data.Pkt, dhcpv4.OptionRelayAgentInformation), + ) + if err != nil { + log.Info("Generating a new transaction id failed, not a problem as we're passing one in, but if this message is showing up a lot then something could be up with github.com/insomniacslk/dhcp") + } + if data.Pkt.OpCode != dhcpv4.OpcodeBootRequest { // TODO(jacobweinstock): dont understand this, found it in an example here: https://github.com/insomniacslk/dhcp/blob/c51060810aaab9c8a0bd1b0fcbf72bc0b91e6427/dhcpv4/server4/server_test.go#L31 + log.V(1).Info("Ignoring packet", "OpCode", data.Pkt.OpCode) + return + } + + if err := dhcp.IsNetbootClient(data.Pkt); err != nil { + log.V(1).Info("Ignoring packet: not from a PXE enabled client", "error", err.Error()) + return + } + + if err := setMessageType(reply, data.Pkt.MessageType()); err != nil { + log.V(1).Info("Ignoring packet", "error", err.Error()) + return + } + + mach := process(data.Pkt) + + // Set option 43 + setOpt43(reply) + + // Set option 97, just copy from the incoming packet + reply.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, data.Pkt.GetOneOption(dhcpv4.OptionClientMachineIdentifier))) + + // set broadcast header to true + // reply.SetBroadcast() + + // Set option 60 + // The PXE spec says the server should identify itself as a PXEClient or HTTPCient + if opt60 := data.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier); strings.HasPrefix(string(opt60), string(dhcp.PXEClient)) { + reply.UpdateOption(dhcpv4.OptClassIdentifier(string(dhcp.PXEClient))) + } else { + reply.UpdateOption(dhcpv4.OptClassIdentifier(string(dhcp.HTTPClient))) + } + + // Set option 54 + opt54 := setOpt54(reply, data.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) + + // add the siaddr (IP address of next server) dhcp packet header to a given packet pkt. + // see https://datatracker.ietf.org/doc/html/rfc2131#section-2 + // without this the pxe client will try to broadcast a request message to port 4011 + reply.ServerIPAddr = opt54 + // probably will want this to be the public IP of the proxyDHCP server + // reply.ServerIPAddr = h.Netboot.IPXEBinServerTFTP.Addr().AsSlice() + + // set sname header + // see https://datatracker.ietf.org/doc/html/rfc2131#section-2 + setSNAME(reply, data.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) + + // set bootfile header + if err := setBootfile(reply, mach, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, h.Netboot.IPXEScriptURL(data.Pkt).String()); err != nil { + log.Info("Ignoring packet", "error", err.Error()) + return + } + // check the backend, if PXE is NOT allowed, set the boot file name to "//not-allowed" + _, n, err := h.Backend.GetByMac(context.Background(), data.Pkt.ClientHWAddr) + if err != nil || (n != nil && !n.AllowNetboot) { + log.V(1).Info("Ignoring packet", "error", err.Error(), "netbootAllowed", n) + return + } + //if !h.Allower.Allow(, mach.mac) { + // rp.BootFileName = fmt.Sprintf("/%v/not-allowed", mach.mac) + //} + + dst := replyDestination(data.Peer, data.Pkt.GatewayIPAddr) + cm := &ipv4.ControlMessage{} + if data.Md != nil { + cm.IfIndex = data.Md.IfIndex + } + // send the DHCP packet + if _, err := conn.WriteTo(reply.ToBytes(), cm, dst); err != nil { + log.Error(err, "failed to send ProxyDHCP offer") + return + } + //log.V(1).Info("DHCP packet received", "pkt", *data.Pkt) + log.Info("Sent ProxyDHCP message", "arch", mach.arch, "userClass", mach.uClass, "receivedMsgType", data.Pkt.MessageType(), "replyMsgType", reply.MessageType(), "unicast", reply.IsUnicast(), "peer", dst, "bootfile", reply.BootFileName) +} + +func setMessageType(reply *dhcpv4.DHCPv4, reqMsg dhcpv4.MessageType) error { + switch mt := reqMsg; mt { + case dhcpv4.MessageTypeDiscover: + reply.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeOffer)) + case dhcpv4.MessageTypeRequest: + reply.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeAck)) + default: + return ErrIgnorePacket{PacketType: mt, Details: "proxyDHCP only responds to Discover or Request DHCP message types"} + } + return nil +} + +// ErrIgnorePacket is for when a DHCP packet should be ignored. +type ErrIgnorePacket struct { + PacketType dhcpv4.MessageType + Details string +} + +// Error returns the string representation of ErrIgnorePacket. +func (e ErrIgnorePacket) Error() string { + return fmt.Sprintf("Ignoring packet: message type %s: details %s", e.PacketType, e.Details) +} + +// processMachine takes a DHCP packet and returns a populated machine struct. +func process(pkt *dhcpv4.DHCPv4) netbootClient { + mach := netbootClient{} + // get option 93 ; arch + fwt := pkt.ClientArch() + if len(fwt) == 0 { + mach.arch = iana.Arch(255) // unassigned/unknown arch + } else { + // a netboot client may have multiple architectures in option 93 + // we will only handle the first one that is not unknown + for _, elem := range fwt { + if !strings.Contains(elem.String(), "unknown") { + // Basic architecture identification, based purely on + // the PXE architecture option. + // https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture + mach.arch = elem + break + } + } + } + + // set option 77 from received packet + mach.uClass = dhcp.UserClass(string(pkt.GetOneOption(dhcpv4.OptionUserClassInformation))) + // set the client type based off of option 60 + opt60 := pkt.GetOneOption(dhcpv4.OptionClassIdentifier) + if strings.HasPrefix(string(opt60), string(dhcp.PXEClient)) { + mach.cType = dhcp.PXEClient + } else if strings.HasPrefix(string(opt60), string(dhcp.HTTPClient)) { + mach.cType = dhcp.HTTPClient + } + mach.mac = pkt.ClientHWAddr + + return mach +} + +// setOpt43 is completely standard PXE: we tell the PXE client to +// bypass all the boot discovery rubbish that PXE supports, +// and just load a file from TFTP. +// TODO(jacobweinstock): add link to intel spec for this needing to be set. +func setOpt43(pkt *dhcpv4.DHCPv4) { + // these are suboptions of option43. ref: https://datatracker.ietf.org/doc/html/rfc2132#section-8.4 + pxe := dhcpv4.Options{ + 6: []byte{8}, // PXE Boot Server Discovery Control - bypass, just boot from filename. + } + // Raspberry PI's need options 9 and 10 of parent option 43. + // The best way at the moment to figure out if a DHCP request is coming from a Raspberry PI is to + // check the MAC address. We could reach out to some external server to tell us if the MAC address should + // use these extra Raspberry PI options but that would require a dependency on some external service and all the trade-offs that + // come with that. TODO: provide doc link for why these options are needed. + // https://udger.com/resources/mac-address-vendor-detail?name=raspberry_pi_foundation + h := strings.ToLower(pkt.ClientHWAddr.String()) + if strings.HasPrefix(h, strings.ToLower("B8:27:EB")) || + strings.HasPrefix(h, strings.ToLower("DC:A6:32")) || + strings.HasPrefix(h, strings.ToLower("E4:5F:01")) { + // TODO document what these hex strings are and why they are needed. + // https://www.raspberrypi.org/documentation/computers/raspberry-pi.html#PXE_OPTION43 + // tested with Raspberry Pi 4 using UEFI from here: https://github.com/pftf/RPi4/releases/tag/v1.31 + // all files were served via a tftp server and lived at the top level dir of the tftp server (i.e tftp://server/) + // "\x00\x00\x11" is equal to NUL(Null), NUL(Null), DC1(Device Control 1) + opt9, _ := hex.DecodeString("00001152617370626572727920506920426f6f74") // "\x00\x00\x11Raspberry Pi Boot" + // "\x0a\x04\x00" is equal to LF(Line Feed), EOT(End of Transmission), NUL(Null) + opt10, _ := hex.DecodeString("00505845") // "\x0a\x04\x00PXE" + pxe[9] = opt9 + pxe[10] = opt10 + } + + pkt.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, pxe.ToBytes())) +} + +// setOpt54 based on option 60. Also return the value for use other locations. +func setOpt54(reply *dhcpv4.DHCPv4, reqOpt60 []byte, tftp net.IP, http net.IP) net.IP { + var opt54 net.IP + if strings.HasPrefix(string(reqOpt60), string(dhcp.HTTPClient)) { + opt54 = http + } else { + opt54 = tftp + } + reply.UpdateOption(dhcpv4.OptServerIdentifier(opt54)) + + return opt54 +} + +// setSNAME sets the server hostname (sname) dhcp header. +func setSNAME(pkt *dhcpv4.DHCPv4, reqOpt60 []byte, tftp net.IP, http net.IP) { + var sname string + if strings.HasPrefix(string(reqOpt60), string(dhcp.HTTPClient)) { + sname = http.String() + } else { + sname = tftp.String() + } + + pkt.ServerHostName = sname +} + +// setBootfile sets the setBootfile (file) dhcp header. see https://datatracker.ietf.org/doc/html/rfc2131#section-2 . +func setBootfile(reply *dhcpv4.DHCPv4, mach netbootClient, tftp netip.AddrPort, ipxe *url.URL, iscript string) error { + // set bootfile header + bin, found := dhcp.ArchToBootFile[mach.arch] + if !found { + return ErrArchNotFound{Arch: mach.arch} + } + var bootfile string + // If a machine is in an ipxe boot loop, it is likely to be that we arent matching on IPXE or Tinkerbell. + // if the "iPXE" user class is found it means we arent in our custom version of ipxe, but because of the option 43 we're setting we need to give a full tftp url from which to boot. + switch { // order matters here. + case mach.uClass == dhcp.Tinkerbell: // this case gets us out of an ipxe boot loop. + bootfile = iscript + case mach.cType == dhcp.HTTPClient: // Check the client type from option 60. + bootfile = fmt.Sprintf("%s/%s/%s", ipxe, mach.mac.String(), bin) + case mach.uClass == dhcp.IPXE: + u := &url.URL{ + Scheme: "tftp", + Host: tftp.String(), + Path: fmt.Sprintf("%v/%v", mach.mac.String(), bin), + } + bootfile = u.String() + default: + bootfile = filepath.Join(mach.mac.String(), bin) + } + reply.BootFileName = bootfile + + return nil +} + +// ErrArchNotFound is for when an PXE client request is an architecture that does not have a matching bootfile. +// See var ArchToBootFile for the look ups. +type ErrArchNotFound struct { + Arch iana.Arch + Detail string +} + +// Error returns the string representation of ErrArchNotFound. +func (e ErrArchNotFound) Error() string { + return fmt.Sprintf("unable to find bootfile for arch %v: details %v", e.Arch, e.Detail) +} + +// replyDestination determines the destination address for the DHCP reply. +// If the giaddr is set, then the reply should be sent to the giaddr. +// Otherwise, the reply should be sent to the direct peer. +// +// From page 22 of https://www.ietf.org/rfc/rfc2131.txt: +// "If the 'giaddr' field in a DHCP message from a client is non-zero, +// the server sends any return messages to the 'DHCP server' port on +// the BOOTP relay agent whose address appears in 'giaddr'.". +func replyDestination(directPeer net.Addr, giaddr net.IP) net.Addr { + if !giaddr.IsUnspecified() && giaddr != nil { + return &net.UDPAddr{IP: giaddr, Port: dhcpv4.ServerPort} + } + + return directPeer +} diff --git a/internal/dhcp/server/dhcp.go b/internal/dhcp/server/dhcp.go new file mode 100644 index 00000000..67b3b2ef --- /dev/null +++ b/internal/dhcp/server/dhcp.go @@ -0,0 +1,112 @@ +// Package dhcp providers UDP listening and serving functionality. +package server + +import ( + "context" + "net" + + "github.com/go-logr/logr" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/dhcpv4/server4" + "github.com/tinkerbell/smee/internal/dhcp/data" + "golang.org/x/net/ipv4" +) + +// Handler is a type that defines the handler function to be called every time a +// valid DHCPv4 message is received +// type Handler func(ctx context.Context, conn net.PacketConn, d data.Packet). +type Handler interface { + Handle(ctx context.Context, conn *ipv4.PacketConn, d data.Packet) +} + +// DHCP represents a DHCPv4 server object. +type DHCP struct { + Conn net.PacketConn + Handlers []Handler + Logger logr.Logger +} + +// Serve serves requests. +func (s *DHCP) Serve(ctx context.Context) error { + go func() { + <-ctx.Done() + _ = s.Close() + }() + s.Logger.Info("Server listening on", "addr", s.Conn.LocalAddr()) + + nConn := ipv4.NewPacketConn(s.Conn) + if err := nConn.SetControlMessage(ipv4.FlagInterface, true); err != nil { + s.Logger.Info("error setting control message", "err", err) + return err + } + + defer func() { + _ = nConn.Close() + }() + for { + // Max UDP packet size is 65535. Max DHCPv4 packet size is 576. An ethernet frame is 1500 bytes. + // We use 4096 as a reasonable buffer size. dhcpv4.FromBytes will handle the rest. + rbuf := make([]byte, 4096) + n, cm, peer, err := nConn.ReadFrom(rbuf) + if err != nil { + select { + case <-ctx.Done(): + return nil + default: + } + s.Logger.Info("error reading from packet conn", "err", err) + return err + } + + m, err := dhcpv4.FromBytes(rbuf[:n]) + if err != nil { + s.Logger.Info("error parsing DHCPv4 request", "err", err) + continue + } + + upeer, ok := peer.(*net.UDPAddr) + if !ok { + s.Logger.Info("not a UDP connection? Peer is", "peer", peer) + continue + } + // Set peer to broadcast if the client did not have an IP. + if upeer.IP == nil || upeer.IP.To4().Equal(net.IPv4zero) { + upeer = &net.UDPAddr{ + IP: net.IPv4bcast, + Port: upeer.Port, + } + } + + var ifName string + if n, err := net.InterfaceByIndex(cm.IfIndex); err == nil { + ifName = n.Name + } + + for _, handler := range s.Handlers { + go handler.Handle(ctx, nConn, data.Packet{Peer: upeer, Pkt: m, Md: &data.Metadata{IfName: ifName, IfIndex: cm.IfIndex}}) + } + } +} + +// Close sends a termination request to the server, and closes the UDP listener. +func (s *DHCP) Close() error { + return s.Conn.Close() +} + +// NewServer initializes and returns a new Server object. +func NewServer(ifname string, addr *net.UDPAddr, handler ...Handler) (*DHCP, error) { + s := &DHCP{ + Handlers: handler, + Logger: logr.Discard(), + } + + if s.Conn == nil { + var err error + conn, err := server4.NewIPv4UDPConn(ifname, addr) + if err != nil { + return nil, err + } + s.Conn = conn + } + return s, nil +} diff --git a/internal/dhcp/dhcp_test.go b/internal/dhcp/server/dhcp_test.go similarity index 99% rename from internal/dhcp/dhcp_test.go rename to internal/dhcp/server/dhcp_test.go index e8b1f252..ef57d052 100644 --- a/internal/dhcp/dhcp_test.go +++ b/internal/dhcp/server/dhcp_test.go @@ -1,4 +1,4 @@ -package dhcp +package server import ( "context" From bce301a610eee345de770b33d604af1f3328e3d5 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Fri, 22 Dec 2023 22:23:23 -0700 Subject: [PATCH 03/17] Fix linting issues Signed-off-by: Jacob Weinstock --- cmd/smee/main.go | 2 +- internal/dhcp/dhcp.go | 6 +-- internal/dhcp/handler/proxy/proxy.go | 63 ++++++++++++++-------------- 3 files changed, 35 insertions(+), 36 deletions(-) diff --git a/cmd/smee/main.go b/cmd/smee/main.go index 8c1c31ed..b90bf3d0 100644 --- a/cmd/smee/main.go +++ b/cmd/smee/main.go @@ -252,7 +252,7 @@ func main() { log.Info("smee is shutting down") } -// func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation.Handler, error) { +// func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation.Handler, error) {. func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*proxy.Handler, error) { // 1. create the handler // 2. create the backend diff --git a/internal/dhcp/dhcp.go b/internal/dhcp/dhcp.go index 1f1630fa..03670b65 100644 --- a/internal/dhcp/dhcp.go +++ b/internal/dhcp/dhcp.go @@ -157,10 +157,10 @@ func IsNetbootClient(pkt *dhcpv4.DHCPv4) error { return err } -func wrapNonNil(err error, format string, a ...any) error { +func wrapNonNil(err error, format string) error { if err == nil { - return fmt.Errorf(format, a...) + return fmt.Errorf(format) } - return fmt.Errorf("%w: %w", err, fmt.Errorf(format, a...)) + return fmt.Errorf("%w: %v", err, format) } diff --git a/internal/dhcp/handler/proxy/proxy.go b/internal/dhcp/handler/proxy/proxy.go index 36ddafb8..260e3513 100644 --- a/internal/dhcp/handler/proxy/proxy.go +++ b/internal/dhcp/handler/proxy/proxy.go @@ -1,5 +1,5 @@ /* - Package proxy implements a DHCP handler that provides proxyDHCP functionality. +Package proxy implements a DHCP handler that provides proxyDHCP functionality. "[A] Proxy DHCP server behaves much like a DHCP server by listening for ordinary DHCP client traffic and responding to certain client requests. However, unlike the @@ -83,51 +83,51 @@ type netbootClient struct { } // Redirection name comes from section 2.5 of http://www.pix.net/software/pxeboot/archive/pxespec.pdf -func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, data data.Packet) { - log := h.Log.WithValues("hwaddr", data.Pkt.ClientHWAddr.String(), "listenAddr", conn.LocalAddr()) - reply, err := dhcpv4.New(dhcpv4.WithReply(data.Pkt), - dhcpv4.WithGatewayIP(data.Pkt.GatewayIPAddr), - dhcpv4.WithOptionCopied(data.Pkt, dhcpv4.OptionRelayAgentInformation), +func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, dp data.Packet) { + log := h.Log.WithValues("hwaddr", dp.Pkt.ClientHWAddr.String(), "listenAddr", conn.LocalAddr()) + reply, err := dhcpv4.New(dhcpv4.WithReply(dp.Pkt), + dhcpv4.WithGatewayIP(dp.Pkt.GatewayIPAddr), + dhcpv4.WithOptionCopied(dp.Pkt, dhcpv4.OptionRelayAgentInformation), ) if err != nil { log.Info("Generating a new transaction id failed, not a problem as we're passing one in, but if this message is showing up a lot then something could be up with github.com/insomniacslk/dhcp") } - if data.Pkt.OpCode != dhcpv4.OpcodeBootRequest { // TODO(jacobweinstock): dont understand this, found it in an example here: https://github.com/insomniacslk/dhcp/blob/c51060810aaab9c8a0bd1b0fcbf72bc0b91e6427/dhcpv4/server4/server_test.go#L31 - log.V(1).Info("Ignoring packet", "OpCode", data.Pkt.OpCode) + if dp.Pkt.OpCode != dhcpv4.OpcodeBootRequest { // TODO(jacobweinstock): dont understand this, found it in an example here: https://github.com/insomniacslk/dhcp/blob/c51060810aaab9c8a0bd1b0fcbf72bc0b91e6427/dhcpv4/server4/server_test.go#L31 + log.V(1).Info("Ignoring packet", "OpCode", dp.Pkt.OpCode) return } - if err := dhcp.IsNetbootClient(data.Pkt); err != nil { + if err := dhcp.IsNetbootClient(dp.Pkt); err != nil { log.V(1).Info("Ignoring packet: not from a PXE enabled client", "error", err.Error()) return } - if err := setMessageType(reply, data.Pkt.MessageType()); err != nil { + if err := setMessageType(reply, dp.Pkt.MessageType()); err != nil { log.V(1).Info("Ignoring packet", "error", err.Error()) return } - mach := process(data.Pkt) + mach := process(dp.Pkt) // Set option 43 setOpt43(reply) // Set option 97, just copy from the incoming packet - reply.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, data.Pkt.GetOneOption(dhcpv4.OptionClientMachineIdentifier))) + reply.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, dp.Pkt.GetOneOption(dhcpv4.OptionClientMachineIdentifier))) // set broadcast header to true // reply.SetBroadcast() // Set option 60 // The PXE spec says the server should identify itself as a PXEClient or HTTPCient - if opt60 := data.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier); strings.HasPrefix(string(opt60), string(dhcp.PXEClient)) { + if opt60 := dp.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier); strings.HasPrefix(string(opt60), string(dhcp.PXEClient)) { reply.UpdateOption(dhcpv4.OptClassIdentifier(string(dhcp.PXEClient))) } else { reply.UpdateOption(dhcpv4.OptClassIdentifier(string(dhcp.HTTPClient))) } // Set option 54 - opt54 := setOpt54(reply, data.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) + opt54 := setOpt54(reply, dp.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) // add the siaddr (IP address of next server) dhcp packet header to a given packet pkt. // see https://datatracker.ietf.org/doc/html/rfc2131#section-2 @@ -138,35 +138,34 @@ func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, data data.P // set sname header // see https://datatracker.ietf.org/doc/html/rfc2131#section-2 - setSNAME(reply, data.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) + setSNAME(reply, dp.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) // set bootfile header - if err := setBootfile(reply, mach, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, h.Netboot.IPXEScriptURL(data.Pkt).String()); err != nil { + if err := setBootfile(reply, mach, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, h.Netboot.IPXEScriptURL(dp.Pkt).String()); err != nil { log.Info("Ignoring packet", "error", err.Error()) return } // check the backend, if PXE is NOT allowed, set the boot file name to "//not-allowed" - _, n, err := h.Backend.GetByMac(context.Background(), data.Pkt.ClientHWAddr) + _, n, err := h.Backend.GetByMac(ctx, dp.Pkt.ClientHWAddr) if err != nil || (n != nil && !n.AllowNetboot) { log.V(1).Info("Ignoring packet", "error", err.Error(), "netbootAllowed", n) return } - //if !h.Allower.Allow(, mach.mac) { + // if !h.Allower.Allow(, mach.mac) { // rp.BootFileName = fmt.Sprintf("/%v/not-allowed", mach.mac) - //} + // } - dst := replyDestination(data.Peer, data.Pkt.GatewayIPAddr) + dst := replyDestination(dp.Peer, dp.Pkt.GatewayIPAddr) cm := &ipv4.ControlMessage{} - if data.Md != nil { - cm.IfIndex = data.Md.IfIndex + if dp.Md != nil { + cm.IfIndex = dp.Md.IfIndex } // send the DHCP packet if _, err := conn.WriteTo(reply.ToBytes(), cm, dst); err != nil { log.Error(err, "failed to send ProxyDHCP offer") return } - //log.V(1).Info("DHCP packet received", "pkt", *data.Pkt) - log.Info("Sent ProxyDHCP message", "arch", mach.arch, "userClass", mach.uClass, "receivedMsgType", data.Pkt.MessageType(), "replyMsgType", reply.MessageType(), "unicast", reply.IsUnicast(), "peer", dst, "bootfile", reply.BootFileName) + log.Info("Sent ProxyDHCP message", "arch", mach.arch, "userClass", mach.uClass, "receivedMsgType", dp.Pkt.MessageType(), "replyMsgType", reply.MessageType(), "unicast", reply.IsUnicast(), "peer", dst, "bootfile", reply.BootFileName) } func setMessageType(reply *dhcpv4.DHCPv4, reqMsg dhcpv4.MessageType) error { @@ -176,19 +175,19 @@ func setMessageType(reply *dhcpv4.DHCPv4, reqMsg dhcpv4.MessageType) error { case dhcpv4.MessageTypeRequest: reply.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeAck)) default: - return ErrIgnorePacket{PacketType: mt, Details: "proxyDHCP only responds to Discover or Request DHCP message types"} + return IgnorePacketError{PacketType: mt, Details: "proxyDHCP only responds to Discover or Request DHCP message types"} } return nil } -// ErrIgnorePacket is for when a DHCP packet should be ignored. -type ErrIgnorePacket struct { +// IgnorePacketError is for when a DHCP packet should be ignored. +type IgnorePacketError struct { PacketType dhcpv4.MessageType Details string } // Error returns the string representation of ErrIgnorePacket. -func (e ErrIgnorePacket) Error() string { +func (e IgnorePacketError) Error() string { return fmt.Sprintf("Ignoring packet: message type %s: details %s", e.PacketType, e.Details) } @@ -291,7 +290,7 @@ func setBootfile(reply *dhcpv4.DHCPv4, mach netbootClient, tftp netip.AddrPort, // set bootfile header bin, found := dhcp.ArchToBootFile[mach.arch] if !found { - return ErrArchNotFound{Arch: mach.arch} + return ArchNotFoundError{Arch: mach.arch} } var bootfile string // If a machine is in an ipxe boot loop, it is likely to be that we arent matching on IPXE or Tinkerbell. @@ -316,15 +315,15 @@ func setBootfile(reply *dhcpv4.DHCPv4, mach netbootClient, tftp netip.AddrPort, return nil } -// ErrArchNotFound is for when an PXE client request is an architecture that does not have a matching bootfile. +// ArchNotFoundError is for when an PXE client request is an architecture that does not have a matching bootfile. // See var ArchToBootFile for the look ups. -type ErrArchNotFound struct { +type ArchNotFoundError struct { Arch iana.Arch Detail string } // Error returns the string representation of ErrArchNotFound. -func (e ErrArchNotFound) Error() string { +func (e ArchNotFoundError) Error() string { return fmt.Sprintf("unable to find bootfile for arch %v: details %v", e.Arch, e.Detail) } From 0f8bd8899ea0a4101ae0e1dfb4b061b0e4f42b77 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Wed, 27 Dec 2023 17:39:16 -0700 Subject: [PATCH 04/17] Create pkg for shared code: The proxy and reservation handler need a good amount of the same code. Signed-off-by: Jacob Weinstock --- docs/Modes.md | 13 + internal/dhcp/dhcp.go | 146 ++++++++- internal/dhcp/dhcp_test.go | 307 ++++++++++++++++++ internal/dhcp/handler/proxy/proxy.go | 285 ++++++---------- internal/dhcp/handler/reservation/handler.go | 78 +---- .../dhcp/handler/reservation/handler_test.go | 53 --- internal/dhcp/handler/reservation/option.go | 130 +------- .../dhcp/handler/reservation/option_test.go | 126 +++---- .../dhcp/handler/reservation/reservation.go | 3 +- 9 files changed, 639 insertions(+), 502 deletions(-) create mode 100644 docs/Modes.md create mode 100644 internal/dhcp/dhcp_test.go diff --git a/docs/Modes.md b/docs/Modes.md new file mode 100644 index 00000000..6c32eea4 --- /dev/null +++ b/docs/Modes.md @@ -0,0 +1,13 @@ +# DHCP Modes + +Smee's DHCP functionality can operate in one of three modes: + +- **DHCP Reservation**: Smee will respond to DHCP requests from clients and provide them with IP and next boot info. This is the default mode. IP info is all reservation based. There must be a corresponding hardware record for the requesting client's MAC address. + +- **proxDHCP**: Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. + +- **DHCP disabled**: Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network and you want to use Smee's TFTP and HTTP functionality. + + + +In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the request to lookup the corresponding hardware. diff --git a/internal/dhcp/dhcp.go b/internal/dhcp/dhcp.go index 03670b65..d1aafc75 100644 --- a/internal/dhcp/dhcp.go +++ b/internal/dhcp/dhcp.go @@ -1,7 +1,11 @@ package dhcp import ( + "encoding/hex" "fmt" + "net" + "net/netip" + "net/url" "strings" "github.com/insomniacslk/dhcp/dhcpv4" @@ -58,6 +62,30 @@ var ArchToBootFile = map[iana.Arch]string{ // ErrUnknownArch is used when the PXE client request is from an unknown architecture. var ErrUnknownArch = fmt.Errorf("could not determine client architecture from option 93") +type Info struct { + Pkt *dhcpv4.DHCPv4 + Arch iana.Arch + Mac net.HardwareAddr + UserClass UserClass + ClientType ClientType + IsNetbootClient error + IPXEBinary string +} + +func NewInfo(pkt *dhcpv4.DHCPv4) Info { + i := Info{Pkt: pkt} + if pkt != nil { + i.Arch = Arch(pkt) + i.Mac = pkt.ClientHWAddr + i.UserClass = i.UserClassFrom() + i.ClientType = i.ClientTypeFrom() + i.IsNetbootClient = IsNetbootClient(pkt) + i.IPXEBinary = i.IPXEBinaryFrom() + } + + return i +} + // Arch returns the Arch of the client pulled from DHCP option 93. func Arch(d *dhcpv4.DHCPv4) iana.Arch { // get option 93 ; arch @@ -84,6 +112,15 @@ func Arch(d *dhcpv4.DHCPv4) iana.Arch { return a } +func (i Info) IPXEBinaryFrom() string { + bin, found := ArchToBootFile[i.Arch] + if !found { + return "" + } + + return bin +} + // String function for clientType. func (c ClientType) String() string { return string(c) @@ -94,6 +131,32 @@ func (u UserClass) String() string { return string(u) } +func (i Info) UserClassFrom() UserClass { + var u UserClass + if i.Pkt != nil { + if val := i.Pkt.Options.Get(dhcpv4.OptionUserClassInformation); val != nil { + u = UserClass(string(val)) + } + } + + return u +} + +func (i Info) ClientTypeFrom() ClientType { + var c ClientType + if i.Pkt != nil { + if val := i.Pkt.Options.Get(dhcpv4.OptionClassIdentifier); val != nil { + if strings.HasPrefix(string(val), HTTPClient.String()) { + c = HTTPClient + } else { + c = PXEClient + } + } + } + + return c +} + // IsNetbootClient returns true if the client is a valid netboot client. // // A valid netboot client will have the following in its DHCP request: @@ -110,30 +173,25 @@ func IsNetbootClient(pkt *dhcpv4.DHCPv4) error { var err error // only response to DISCOVER and REQUEST packets if pkt.MessageType() != dhcpv4.MessageTypeDiscover && pkt.MessageType() != dhcpv4.MessageTypeRequest { - // h.Log.Info("not a netboot client", "reason", "message type must be either Discover or Request", "mac", pkt.ClientHWAddr.String(), "message type", pkt.MessageType()) err = wrapNonNil(err, "message type must be either Discover or Request") } // option 60 must be set if !pkt.Options.Has(dhcpv4.OptionClassIdentifier) { - // h.Log.Info("not a netboot client", "reason", "option 60 not set", "mac", pkt.ClientHWAddr.String()) err = wrapNonNil(err, "option 60 not set") } // option 60 must start with PXEClient or HTTPClient opt60 := pkt.GetOneOption(dhcpv4.OptionClassIdentifier) if !strings.HasPrefix(string(opt60), string(PXEClient)) && !strings.HasPrefix(string(opt60), string(HTTPClient)) { - // h.Log.Info("not a netboot client", "reason", "option 60 not PXEClient or HTTPClient", "mac", pkt.ClientHWAddr.String(), "option 60", string(opt60)) err = wrapNonNil(err, "option 60 not PXEClient or HTTPClient") } // option 93 must be set if !pkt.Options.Has(dhcpv4.OptionClientSystemArchitectureType) { - // h.Log.Info("not a netboot client", "reason", "option 93 not set", "mac", pkt.ClientHWAddr.String()) err = wrapNonNil(err, "option 93 not set") } // option 94 must be set if !pkt.Options.Has(dhcpv4.OptionClientNetworkInterfaceIdentifier) { - // h.Log.Info("not a netboot client", "reason", "option 94 not set", "mac", pkt.ClientHWAddr.String()) err = wrapNonNil(err, "option 94 not set") } @@ -164,3 +222,81 @@ func wrapNonNil(err error, format string) error { return fmt.Errorf("%w: %v", err, format) } + +// Bootfile returns the calculated dhcp header: "file" value. see https://datatracker.ietf.org/doc/html/rfc2131#section-2 . +func (i Info) Bootfile(customUC UserClass, ipxeScript, ipxeHTTPBinServer *url.URL, ipxeTFTPBinServer netip.AddrPort) string { + bootfile := "/no-ipxe-script-defined" + + // If a machine is in an ipxe boot loop, it is likely to be that we aren't matching on IPXE or Tinkerbell userclass (option 77). + switch { // order matters here. + case i.UserClass == Tinkerbell, (customUC != "" && i.UserClass == customUC): // this case gets us out of an ipxe boot loop. + if ipxeScript != nil { + bootfile = ipxeScript.String() + } + case i.ClientType == HTTPClient: // Check the client type from option 60. + if ipxeHTTPBinServer != nil { + paths := []string{i.IPXEBinary} + if i.Mac != nil { + paths = append([]string{i.Mac.String()}, paths...) + } + bootfile = ipxeHTTPBinServer.JoinPath(paths...).String() + } + case i.UserClass == IPXE: // if the "iPXE" user class is found it means we aren't in our custom version of ipxe, but because of the option 43 we're setting we need to give a full tftp url from which to boot. + t := url.URL{ + Scheme: "tftp", + Host: ipxeTFTPBinServer.String(), + } + paths := []string{i.IPXEBinary} + if i.Mac != nil { + paths = append([]string{i.Mac.String()}, paths...) + } + bootfile = t.JoinPath(paths...).String() + default: + if i.IPXEBinary != "" { + bootfile = i.IPXEBinary + } + } + + return bootfile +} + +// NextServer returns the calculated dhcp header (ServerIPAddr): "siaddr" value. see https://datatracker.ietf.org/doc/html/rfc2131#section-2 . +func (i Info) NextServer(ipxeHTTPBinServer *url.URL, ipxeTFTPBinServer netip.AddrPort) net.IP { + var nextServer net.IP + + // If a machine is in an ipxe boot loop, it is likely to be that we aren't matching on IPXE or Tinkerbell userclass (option 77). + switch { // order matters here. + case i.ClientType == HTTPClient: // Check the client type from option 60. + if ipxeHTTPBinServer != nil { + nextServer = net.ParseIP(ipxeHTTPBinServer.Hostname()) + } + case i.UserClass == IPXE: // if the "iPXE" user class is found it means we aren't in our custom version of ipxe, but because of the option 43 we're setting we need to give a full tftp url from which to boot. + nextServer = net.IP(ipxeTFTPBinServer.Addr().AsSlice()) + default: + nextServer = net.IP(ipxeTFTPBinServer.Addr().AsSlice()) + } + + return nextServer +} + +// AddRPIOpt43 adds the Raspberry PI required option43 sub options to an existing opt 43. +func (i Info) AddRPIOpt43(opts dhcpv4.Options) []byte { + // these are suboptions of option43. ref: https://datatracker.ietf.org/doc/html/rfc2132#section-8.4 + h := strings.ToLower(i.Mac.String()) + if strings.HasPrefix(h, strings.ToLower("B8:27:EB")) || + strings.HasPrefix(h, strings.ToLower("DC:A6:32")) || + strings.HasPrefix(h, strings.ToLower("E4:5F:01")) { + // TODO document what these hex strings are and why they are needed. + // https://www.raspberrypi.org/documentation/computers/raspberry-pi.html#PXE_OPTION43 + // tested with Raspberry Pi 4 using UEFI from here: https://github.com/pftf/RPi4/releases/tag/v1.31 + // all files were served via a tftp server and lived at the top level dir of the tftp server (i.e tftp://server/) + // "\x00\x00\x11" is equal to NUL(Null), NUL(Null), DC1(Device Control 1) + opt9, _ := hex.DecodeString("00001152617370626572727920506920426f6f74") // "\x00\x00\x11Raspberry Pi Boot" + opts[9] = opt9 + // "\x0a\x04\x00" is equal to LF(Line Feed), EOT(End of Transmission), NUL(Null) + opt10, _ := hex.DecodeString("00505845") // "\x0a\x04\x00PXE" + opts[10] = opt10 + } + + return opts.ToBytes() +} diff --git a/internal/dhcp/dhcp_test.go b/internal/dhcp/dhcp_test.go new file mode 100644 index 00000000..ee3ca83d --- /dev/null +++ b/internal/dhcp/dhcp_test.go @@ -0,0 +1,307 @@ +package dhcp + +import ( + "encoding/hex" + "errors" + "net" + "net/netip" + "net/url" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/insomniacslk/dhcp/iana" +) + +const ( + examplePXEClient = "PXEClient:Arch:00007:UNDI:003001" + exampleHTTPClient = "HTTPClient:Arch:00016:UNDI:003001" +) + +func TestNewInfo(t *testing.T) { + tests := map[string]struct { + pkt *dhcpv4.DHCPv4 + want Info + }{ + "valid http client": { + pkt: &dhcpv4.DHCPv4{ + ClientIPAddr: []byte{0x00, 0x00, 0x00, 0x00}, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClientArch(iana.EFI_X86_64_HTTP), + dhcpv4.OptUserClass(Tinkerbell.String()), + dhcpv4.OptClassIdentifier(exampleHTTPClient), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x05, 0x06, 0x07}), + ), + }, + want: Info{ + Arch: iana.EFI_X86_64_HTTP, + Mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + UserClass: Tinkerbell, + ClientType: HTTPClient, + IsNetbootClient: nil, + IPXEBinary: "ipxe.efi", + }, + }, + "arch not found": { + pkt: &dhcpv4.DHCPv4{ + ClientIPAddr: []byte{0x00, 0x00, 0x00, 0x00}, + ClientHWAddr: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClientArch(iana.Arch(255)), + dhcpv4.OptClassIdentifier(examplePXEClient), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x05, 0x06, 0x07}), + ), + }, + want: Info{ + Arch: iana.Arch(255), + Mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + ClientType: PXEClient, + }, + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := NewInfo(tt.pkt) + if diff := cmp.Diff(tt.want, got, cmpopts.IgnoreFields(Info{}, "Pkt")); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestArch(t *testing.T) { + tests := map[string]struct { + pkt *dhcpv4.DHCPv4 + want iana.Arch + }{ + "found": { + pkt: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptClientArch(iana.INTEL_X86PC))}, + want: iana.INTEL_X86PC, + }, + "unknown": { + pkt: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptClientArch(iana.Arch(255)))}, + want: iana.Arch(255), + }, + "unknown: opt 93 len 0": { + pkt: &dhcpv4.DHCPv4{}, + want: iana.Arch(255), + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := Arch(tt.pkt) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestIsNetbootClient(t *testing.T) { + tests := map[string]struct { + input *dhcpv4.DHCPv4 + want error + }{ + "fail invalid message type": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptMessageType(dhcpv4.MessageTypeInform))}, want: errors.New("")}, + "fail no opt60": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover))}, want: errors.New("")}, + "fail bad opt60": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("BadClient"), + )}, want: errors.New("")}, + "fail no opt93": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + )}, want: errors.New("")}, + "fail no opt94": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + )}, want: errors.New("")}, + "fail invalid opt97[0] != 0": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05}), + )}, want: errors.New("")}, + "fail invalid len(opt97)": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x01, 0x02}), + )}, want: errors.New("")}, + "success len(opt97) == 0": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( + dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), + dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), + dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{}), + )}, want: nil}, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + if err := IsNetbootClient(tt.input); (err == nil) != (tt.want == nil) { + t.Errorf("isNetbootClient() = %v, want %v", err, tt.want) + } + }) + } +} + +func TestBootfile(t *testing.T) { + type args struct { + customUC UserClass + ipxeTFTPBinServer netip.AddrPort + ipxeScript *url.URL + ipxeHTTPBinServer *url.URL + } + tests := map[string]struct { + info Info + args args + want string + }{ + "ipxe script": { + info: Info{ + UserClass: Tinkerbell, + }, + args: args{ + ipxeScript: &url.URL{Path: "/ipxe-script"}, + }, + want: "/ipxe-script", + }, + "http client": { + info: Info{ + ClientType: HTTPClient, + Mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + IPXEBinary: "ipxe.efi", + }, + args: args{ + ipxeHTTPBinServer: &url.URL{Scheme: "http", Host: "1.2.3.4:8080"}, + }, + want: "http://1.2.3.4:8080/01:02:03:04:05:06/ipxe.efi", + }, + "firmware ipxe": { + info: Info{ + UserClass: IPXE, + Mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + IPXEBinary: "undionly.kpxe", + }, + args: args{ + ipxeTFTPBinServer: netip.MustParseAddrPort("1.2.3.4:69"), + }, + want: "tftp://1.2.3.4:69/01:02:03:04:05:06/undionly.kpxe", + }, + "no user class": { + info: Info{ + Mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + IPXEBinary: "undionly.kpxe", + }, + want: "undionly.kpxe", + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := tt.info.Bootfile(tt.args.customUC, tt.args.ipxeScript, tt.args.ipxeHTTPBinServer, tt.args.ipxeTFTPBinServer) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestNextServer(t *testing.T) { + type args struct { + ipxeTFTPBinServer netip.AddrPort + ipxeHTTPBinServer *url.URL + } + tests := map[string]struct { + info Info + args args + want net.IP + }{ + "http client": { + info: Info{ + ClientType: HTTPClient, + }, + args: args{ + ipxeHTTPBinServer: &url.URL{Scheme: "http", Host: "1.2.3.4:8989"}, + }, + want: net.ParseIP("1.2.3.4"), + }, + "firmware ipxe": { + info: Info{ + UserClass: IPXE, + }, + args: args{ + ipxeTFTPBinServer: netip.MustParseAddrPort("1.2.3.4:69"), + }, + want: net.ParseIP("1.2.3.4"), + }, + "no user class": { + info: Info{}, + args: args{ + ipxeTFTPBinServer: netip.MustParseAddrPort("1.2.3.4:69"), + }, + want: net.ParseIP("1.2.3.4"), + }, + } + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := tt.info.NextServer(tt.args.ipxeHTTPBinServer, tt.args.ipxeTFTPBinServer) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestOpt43(t *testing.T) { + rpi9, _ := hex.DecodeString("00001152617370626572727920506920426f6f74") + rpi10, _ := hex.DecodeString("00505845") + + tests := map[string]struct { + info Info + opts dhcpv4.Options + want []byte + }{ + "not a raspberry pi": { + info: Info{ + Mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + }, + opts: dhcpv4.Options{}, + want: dhcpv4.Options{}.ToBytes(), + }, + "raspberry pi": { + info: Info{ + Mac: net.HardwareAddr{0xb8, 0x27, 0xeb, 0x00, 0x00, 0x00}, + }, + opts: dhcpv4.Options{}, + want: dhcpv4.Options{ + 9: rpi9, + 10: rpi10, + }.ToBytes(), + }, + } + + for name, tt := range tests { + t.Run(name, func(t *testing.T) { + got := tt.info.AddRPIOpt43(tt.opts) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestUserClassString(t *testing.T) { + u := UserClass("test") + if diff := cmp.Diff("test", u.String()); diff != "" { + t.Fatal(diff) + } +} diff --git a/internal/dhcp/handler/proxy/proxy.go b/internal/dhcp/handler/proxy/proxy.go index 260e3513..608d98bf 100644 --- a/internal/dhcp/handler/proxy/proxy.go +++ b/internal/dhcp/handler/proxy/proxy.go @@ -15,23 +15,26 @@ package proxy import ( "context" - "encoding/hex" + "errors" "fmt" "net" "net/netip" "net/url" - "path/filepath" - "strings" "github.com/go-logr/logr" "github.com/insomniacslk/dhcp/dhcpv4" - "github.com/insomniacslk/dhcp/iana" "github.com/tinkerbell/smee/internal/dhcp" "github.com/tinkerbell/smee/internal/dhcp/data" "github.com/tinkerbell/smee/internal/dhcp/handler" + oteldhcp "github.com/tinkerbell/smee/internal/dhcp/otel" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" "golang.org/x/net/ipv4" ) +const tracerName = "github.com/tinkerbell/smee/internal/dhcp/handler/proxy" + // Handler holds the configuration details for the running the DHCP server. type Handler struct { // Backend is the backend to use for getting DHCP data. @@ -74,31 +77,51 @@ type Netboot struct { UserClass dhcp.UserClass } -// netbootClient describes a device that is requesting a network boot. -type netbootClient struct { - mac net.HardwareAddr - arch iana.Arch - uClass dhcp.UserClass - cType dhcp.ClientType -} - // Redirection name comes from section 2.5 of http://www.pix.net/software/pxeboot/archive/pxespec.pdf func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, dp data.Packet) { - log := h.Log.WithValues("hwaddr", dp.Pkt.ClientHWAddr.String(), "listenAddr", conn.LocalAddr()) - reply, err := dhcpv4.New(dhcpv4.WithReply(dp.Pkt), - dhcpv4.WithGatewayIP(dp.Pkt.GatewayIPAddr), - dhcpv4.WithOptionCopied(dp.Pkt, dhcpv4.OptionRelayAgentInformation), - ) - if err != nil { - log.Info("Generating a new transaction id failed, not a problem as we're passing one in, but if this message is showing up a lot then something could be up with github.com/insomniacslk/dhcp") + // validations + if dp.Pkt == nil { + h.Log.Error(errors.New("incoming packet is nil"), "not able to respond when the incoming packet is nil") + return } - if dp.Pkt.OpCode != dhcpv4.OpcodeBootRequest { // TODO(jacobweinstock): dont understand this, found it in an example here: https://github.com/insomniacslk/dhcp/blob/c51060810aaab9c8a0bd1b0fcbf72bc0b91e6427/dhcpv4/server4/server_test.go#L31 - log.V(1).Info("Ignoring packet", "OpCode", dp.Pkt.OpCode) + upeer, ok := dp.Peer.(*net.UDPAddr) + if !ok { + h.Log.Error(errors.New("peer is not a UDP connection"), "not able to respond when the peer is not a UDP connection") + return + } + if upeer == nil { + h.Log.Error(errors.New("peer is nil"), "not able to respond when the peer is nil") + return + } + if conn == nil { + h.Log.Error(errors.New("connection is nil"), "not able to respond when the connection is nil") return } - if err := dhcp.IsNetbootClient(dp.Pkt); err != nil { - log.V(1).Info("Ignoring packet: not from a PXE enabled client", "error", err.Error()) + var ifName string + if dp.Md != nil { + ifName = dp.Md.IfName + } + log := h.Log.WithValues("mac", dp.Pkt.ClientHWAddr.String(), "xid", dp.Pkt.TransactionID.String(), "interface", ifName) + tracer := otel.Tracer(tracerName) + var span trace.Span + ctx, span = tracer.Start( + ctx, + fmt.Sprintf("DHCP Packet Received: %v", dp.Pkt.MessageType().String()), + trace.WithAttributes(h.encodeToAttributes(dp.Pkt, "request")...), + trace.WithAttributes(attribute.String("DHCP.peer", dp.Peer.String())), + trace.WithAttributes(attribute.String("DHCP.server.ifname", ifName)), + ) + + defer span.End() + + // We ignore the error here because: + // 1. it's only non-nil if the generation of a transaction id (XID) fails. + // 2. We always use the clients transaction id (XID) in responses. See dhcpv4.WithReply(). + reply, _ := dhcpv4.NewReplyFromRequest(dp.Pkt) + + if dp.Pkt.OpCode != dhcpv4.OpcodeBootRequest { // TODO(jacobweinstock): dont understand this, found it in an example here: https://github.com/insomniacslk/dhcp/blob/c51060810aaab9c8a0bd1b0fcbf72bc0b91e6427/dhcpv4/server4/server_test.go#L31 + log.V(1).Info("Ignoring packet", "OpCode", dp.Pkt.OpCode) return } @@ -107,65 +130,83 @@ func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, dp data.Pac return } - mach := process(dp.Pkt) - - // Set option 43 - setOpt43(reply) - - // Set option 97, just copy from the incoming packet + // Set option 97 reply.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, dp.Pkt.GetOneOption(dhcpv4.OptionClientMachineIdentifier))) - // set broadcast header to true - // reply.SetBroadcast() + i := dhcp.NewInfo(dp.Pkt) - // Set option 60 - // The PXE spec says the server should identify itself as a PXEClient or HTTPCient - if opt60 := dp.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier); strings.HasPrefix(string(opt60), string(dhcp.PXEClient)) { - reply.UpdateOption(dhcpv4.OptClassIdentifier(string(dhcp.PXEClient))) - } else { - reply.UpdateOption(dhcpv4.OptClassIdentifier(string(dhcp.HTTPClient))) + if err := i.IsNetbootClient; err != nil { + log.V(1).Info("Ignoring packet: not from a PXE enabled client", "error", err.Error()) + return + } + if i.IPXEBinary == "" { + log.V(1).Info("Ignoring packet: no iPXE binary was able to be determined") + return } - // Set option 54 - opt54 := setOpt54(reply, dp.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) + // Set option 43 + opts := dhcpv4.Options{6: []byte{8}} // PXE Boot Server Discovery Control - bypass, just boot from dhcp header: bootfile. No need to set opt for tftp server address. + reply.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, i.AddRPIOpt43(opts))) + + // Set option 60 + // The PXE spec says the server should identify itself as a PXEClient or HTTPClient + reply.UpdateOption(dhcpv4.OptClassIdentifier(i.ClientTypeFrom().String())) + // Set option 54, without this the pxe client will try to broadcast a request message to port 4011 for the ipxe binary. only found to be needed for PXEClient but not prohibitive for HTTPClient. + // probably will want this to be the public IP of the proxyDHCP server + ns := i.NextServer(h.Netboot.IPXEBinServerHTTP, h.Netboot.IPXEBinServerTFTP) + reply.UpdateOption(dhcpv4.OptServerIdentifier(ns)) // add the siaddr (IP address of next server) dhcp packet header to a given packet pkt. // see https://datatracker.ietf.org/doc/html/rfc2131#section-2 - // without this the pxe client will try to broadcast a request message to port 4011 - reply.ServerIPAddr = opt54 - // probably will want this to be the public IP of the proxyDHCP server - // reply.ServerIPAddr = h.Netboot.IPXEBinServerTFTP.Addr().AsSlice() + // without this the pxe client will try to broadcast a request message to port 4011 for the ipxe script. The value doesnt seem to matter. + reply.ServerIPAddr = ns // set sname header // see https://datatracker.ietf.org/doc/html/rfc2131#section-2 - setSNAME(reply, dp.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) + reply.ServerHostName = ns.String() + // setSNAME(reply, dp.Pkt.GetOneOption(dhcpv4.OptionClassIdentifier), h.Netboot.IPXEBinServerTFTP.Addr().AsSlice(), net.ParseIP(h.Netboot.IPXEBinServerHTTP.Hostname())) // set bootfile header - if err := setBootfile(reply, mach, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, h.Netboot.IPXEScriptURL(dp.Pkt).String()); err != nil { - log.Info("Ignoring packet", "error", err.Error()) - return - } + reply.BootFileName = i.Bootfile("", h.Netboot.IPXEScriptURL(dp.Pkt), h.Netboot.IPXEBinServerHTTP, h.Netboot.IPXEBinServerTFTP) + // check the backend, if PXE is NOT allowed, set the boot file name to "//not-allowed" _, n, err := h.Backend.GetByMac(ctx, dp.Pkt.ClientHWAddr) if err != nil || (n != nil && !n.AllowNetboot) { log.V(1).Info("Ignoring packet", "error", err.Error(), "netbootAllowed", n) return } - // if !h.Allower.Allow(, mach.mac) { - // rp.BootFileName = fmt.Sprintf("/%v/not-allowed", mach.mac) - // } + log.Info( + "received DHCP packet", + "type", dp.Pkt.MessageType().String(), + "clientType", i.ClientTypeFrom().String(), + "userClass", i.UserClassFrom().String(), + ) dst := replyDestination(dp.Peer, dp.Pkt.GatewayIPAddr) cm := &ipv4.ControlMessage{} if dp.Md != nil { cm.IfIndex = dp.Md.IfIndex } + log = log.WithValues( + "destination", dst.String(), + "bootFileName", reply.BootFileName, + "nextServer", reply.ServerIPAddr.String(), + "messageType", reply.MessageType().String(), + "serverHostname", reply.ServerHostName, + ) // send the DHCP packet if _, err := conn.WriteTo(reply.ToBytes(), cm, dst); err != nil { - log.Error(err, "failed to send ProxyDHCP offer") + log.Error(err, "failed to send ProxyDHCP response") return } - log.Info("Sent ProxyDHCP message", "arch", mach.arch, "userClass", mach.uClass, "receivedMsgType", dp.Pkt.MessageType(), "replyMsgType", reply.MessageType(), "unicast", reply.IsUnicast(), "peer", dst, "bootfile", reply.BootFileName) + log.Info("Sent ProxyDHCP response") +} + +// encodeToAttributes takes a DHCP packet and returns opentelemetry key/value attributes. +func (h *Handler) encodeToAttributes(d *dhcpv4.DHCPv4, namespace string) []attribute.KeyValue { + a := &oteldhcp.Encoder{Log: h.Log} + + return a.Encode(d, namespace, oteldhcp.AllEncoders()...) } func setMessageType(reply *dhcpv4.DHCPv4, reqMsg dhcpv4.MessageType) error { @@ -175,7 +216,7 @@ func setMessageType(reply *dhcpv4.DHCPv4, reqMsg dhcpv4.MessageType) error { case dhcpv4.MessageTypeRequest: reply.UpdateOption(dhcpv4.OptMessageType(dhcpv4.MessageTypeAck)) default: - return IgnorePacketError{PacketType: mt, Details: "proxyDHCP only responds to Discover or Request DHCP message types"} + return IgnorePacketError{PacketType: mt, Details: "proxyDHCP only responds to Discover or Request message types"} } return nil } @@ -191,142 +232,6 @@ func (e IgnorePacketError) Error() string { return fmt.Sprintf("Ignoring packet: message type %s: details %s", e.PacketType, e.Details) } -// processMachine takes a DHCP packet and returns a populated machine struct. -func process(pkt *dhcpv4.DHCPv4) netbootClient { - mach := netbootClient{} - // get option 93 ; arch - fwt := pkt.ClientArch() - if len(fwt) == 0 { - mach.arch = iana.Arch(255) // unassigned/unknown arch - } else { - // a netboot client may have multiple architectures in option 93 - // we will only handle the first one that is not unknown - for _, elem := range fwt { - if !strings.Contains(elem.String(), "unknown") { - // Basic architecture identification, based purely on - // the PXE architecture option. - // https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture - mach.arch = elem - break - } - } - } - - // set option 77 from received packet - mach.uClass = dhcp.UserClass(string(pkt.GetOneOption(dhcpv4.OptionUserClassInformation))) - // set the client type based off of option 60 - opt60 := pkt.GetOneOption(dhcpv4.OptionClassIdentifier) - if strings.HasPrefix(string(opt60), string(dhcp.PXEClient)) { - mach.cType = dhcp.PXEClient - } else if strings.HasPrefix(string(opt60), string(dhcp.HTTPClient)) { - mach.cType = dhcp.HTTPClient - } - mach.mac = pkt.ClientHWAddr - - return mach -} - -// setOpt43 is completely standard PXE: we tell the PXE client to -// bypass all the boot discovery rubbish that PXE supports, -// and just load a file from TFTP. -// TODO(jacobweinstock): add link to intel spec for this needing to be set. -func setOpt43(pkt *dhcpv4.DHCPv4) { - // these are suboptions of option43. ref: https://datatracker.ietf.org/doc/html/rfc2132#section-8.4 - pxe := dhcpv4.Options{ - 6: []byte{8}, // PXE Boot Server Discovery Control - bypass, just boot from filename. - } - // Raspberry PI's need options 9 and 10 of parent option 43. - // The best way at the moment to figure out if a DHCP request is coming from a Raspberry PI is to - // check the MAC address. We could reach out to some external server to tell us if the MAC address should - // use these extra Raspberry PI options but that would require a dependency on some external service and all the trade-offs that - // come with that. TODO: provide doc link for why these options are needed. - // https://udger.com/resources/mac-address-vendor-detail?name=raspberry_pi_foundation - h := strings.ToLower(pkt.ClientHWAddr.String()) - if strings.HasPrefix(h, strings.ToLower("B8:27:EB")) || - strings.HasPrefix(h, strings.ToLower("DC:A6:32")) || - strings.HasPrefix(h, strings.ToLower("E4:5F:01")) { - // TODO document what these hex strings are and why they are needed. - // https://www.raspberrypi.org/documentation/computers/raspberry-pi.html#PXE_OPTION43 - // tested with Raspberry Pi 4 using UEFI from here: https://github.com/pftf/RPi4/releases/tag/v1.31 - // all files were served via a tftp server and lived at the top level dir of the tftp server (i.e tftp://server/) - // "\x00\x00\x11" is equal to NUL(Null), NUL(Null), DC1(Device Control 1) - opt9, _ := hex.DecodeString("00001152617370626572727920506920426f6f74") // "\x00\x00\x11Raspberry Pi Boot" - // "\x0a\x04\x00" is equal to LF(Line Feed), EOT(End of Transmission), NUL(Null) - opt10, _ := hex.DecodeString("00505845") // "\x0a\x04\x00PXE" - pxe[9] = opt9 - pxe[10] = opt10 - } - - pkt.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, pxe.ToBytes())) -} - -// setOpt54 based on option 60. Also return the value for use other locations. -func setOpt54(reply *dhcpv4.DHCPv4, reqOpt60 []byte, tftp net.IP, http net.IP) net.IP { - var opt54 net.IP - if strings.HasPrefix(string(reqOpt60), string(dhcp.HTTPClient)) { - opt54 = http - } else { - opt54 = tftp - } - reply.UpdateOption(dhcpv4.OptServerIdentifier(opt54)) - - return opt54 -} - -// setSNAME sets the server hostname (sname) dhcp header. -func setSNAME(pkt *dhcpv4.DHCPv4, reqOpt60 []byte, tftp net.IP, http net.IP) { - var sname string - if strings.HasPrefix(string(reqOpt60), string(dhcp.HTTPClient)) { - sname = http.String() - } else { - sname = tftp.String() - } - - pkt.ServerHostName = sname -} - -// setBootfile sets the setBootfile (file) dhcp header. see https://datatracker.ietf.org/doc/html/rfc2131#section-2 . -func setBootfile(reply *dhcpv4.DHCPv4, mach netbootClient, tftp netip.AddrPort, ipxe *url.URL, iscript string) error { - // set bootfile header - bin, found := dhcp.ArchToBootFile[mach.arch] - if !found { - return ArchNotFoundError{Arch: mach.arch} - } - var bootfile string - // If a machine is in an ipxe boot loop, it is likely to be that we arent matching on IPXE or Tinkerbell. - // if the "iPXE" user class is found it means we arent in our custom version of ipxe, but because of the option 43 we're setting we need to give a full tftp url from which to boot. - switch { // order matters here. - case mach.uClass == dhcp.Tinkerbell: // this case gets us out of an ipxe boot loop. - bootfile = iscript - case mach.cType == dhcp.HTTPClient: // Check the client type from option 60. - bootfile = fmt.Sprintf("%s/%s/%s", ipxe, mach.mac.String(), bin) - case mach.uClass == dhcp.IPXE: - u := &url.URL{ - Scheme: "tftp", - Host: tftp.String(), - Path: fmt.Sprintf("%v/%v", mach.mac.String(), bin), - } - bootfile = u.String() - default: - bootfile = filepath.Join(mach.mac.String(), bin) - } - reply.BootFileName = bootfile - - return nil -} - -// ArchNotFoundError is for when an PXE client request is an architecture that does not have a matching bootfile. -// See var ArchToBootFile for the look ups. -type ArchNotFoundError struct { - Arch iana.Arch - Detail string -} - -// Error returns the string representation of ErrArchNotFound. -func (e ArchNotFoundError) Error() string { - return fmt.Sprintf("unable to find bootfile for arch %v: details %v", e.Arch, e.Detail) -} - // replyDestination determines the destination address for the DHCP reply. // If the giaddr is set, then the reply should be sent to the giaddr. // Otherwise, the reply should be sent to the direct peer. diff --git a/internal/dhcp/handler/reservation/handler.go b/internal/dhcp/handler/reservation/handler.go index b78996ef..ec0bb4c8 100644 --- a/internal/dhcp/handler/reservation/handler.go +++ b/internal/dhcp/handler/reservation/handler.go @@ -5,10 +5,10 @@ import ( "errors" "fmt" "net" - "strings" "github.com/go-logr/logr" "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/tinkerbell/smee/internal/dhcp" "github.com/tinkerbell/smee/internal/dhcp/data" oteldhcp "github.com/tinkerbell/smee/internal/dhcp/otel" "go.opentelemetry.io/otel" @@ -191,83 +191,17 @@ func (h *Handler) updateMsg(ctx context.Context, pkt *dhcpv4.DHCPv4, d *data.DHC } mods = append(mods, h.setDHCPOpts(ctx, pkt, d)...) - if h.Netboot.Enabled && h.isNetbootClient(pkt) == nil { + if h.Netboot.Enabled && dhcp.IsNetbootClient(pkt) == nil { mods = append(mods, h.setNetworkBootOpts(ctx, pkt, n)) } - reply, err := dhcpv4.NewReplyFromRequest(pkt, mods...) - if err != nil { - return nil - } + // We ignore the error here because: + // 1. it's only non-nil if the generation of a transaction id (XID) fails. + // 2. We always use the clients transaction id (XID) in responses. See dhcpv4.WithReply(). + reply, _ := dhcpv4.NewReplyFromRequest(pkt, mods...) return reply } -// isNetbootClient returns true if the client is a valid netboot client. -// -// A valid netboot client will have the following in its DHCP request: -// 1. is a DHCP discovery/request message type. -// 2. option 93 is set. -// 3. option 94 is set. -// 4. option 97 is correct length. -// 5. option 60 is set with this format: "PXEClient:Arch:xxxxx:UNDI:yyyzzz" or "HTTPClient:Arch:xxxxx:UNDI:yyyzzz". -// -// See: http://www.pix.net/software/pxeboot/archive/pxespec.pdf -// -// See: https://www.rfc-editor.org/rfc/rfc4578.html -func (h *Handler) isNetbootClient(pkt *dhcpv4.DHCPv4) error { - h.setDefaults() - var err error - // only response to DISCOVER and REQUEST packets - if pkt.MessageType() != dhcpv4.MessageTypeDiscover && pkt.MessageType() != dhcpv4.MessageTypeRequest { - // h.Log.Info("not a netboot client", "reason", "message type must be either Discover or Request", "mac", pkt.ClientHWAddr.String(), "message type", pkt.MessageType()) - err = errors.New("message type must be either Discover or Request") - } - // option 60 must be set - if !pkt.Options.Has(dhcpv4.OptionClassIdentifier) { - // h.Log.Info("not a netboot client", "reason", "option 60 not set", "mac", pkt.ClientHWAddr.String()) - err = fmt.Errorf("%w: option 60 not set", err) - } - // option 60 must start with PXEClient or HTTPClient - opt60 := pkt.GetOneOption(dhcpv4.OptionClassIdentifier) - if !strings.HasPrefix(string(opt60), string(pxeClient)) && !strings.HasPrefix(string(opt60), string(httpClient)) { - // h.Log.Info("not a netboot client", "reason", "option 60 not PXEClient or HTTPClient", "mac", pkt.ClientHWAddr.String(), "option 60", string(opt60)) - err = fmt.Errorf("%w: option 60 not PXEClient or HTTPClient", err) - } - - // option 93 must be set - if !pkt.Options.Has(dhcpv4.OptionClientSystemArchitectureType) { - // h.Log.Info("not a netboot client", "reason", "option 93 not set", "mac", pkt.ClientHWAddr.String()) - err = fmt.Errorf("%w: option 93 not set", err) - } - - // option 94 must be set - if !pkt.Options.Has(dhcpv4.OptionClientNetworkInterfaceIdentifier) { - // h.Log.Info("not a netboot client", "reason", "option 94 not set", "mac", pkt.ClientHWAddr.String()) - err = fmt.Errorf("%w: option 94 not set", err) - } - - // option 97 must be have correct length or not be set - guid := pkt.GetOneOption(dhcpv4.OptionClientMachineIdentifier) - switch len(guid) { - case 0: - // A missing GUID is invalid according to the spec, however - // there are PXE ROMs in the wild that omit the GUID and still - // expect to boot. The only thing we do with the GUID is - // mirror it back to the client if it's there, so we might as - // well accept these buggy ROMs. - case 17: - if guid[0] != 0 { - h.Log.Info("not a netboot client", "reason", "option 97 does not start with 0", "mac", pkt.ClientHWAddr.String(), "option 97", string(guid)) - err = fmt.Errorf("%w: option 97 does not start with 0", err) - } - default: - h.Log.Info("not a netboot client", "reason", "option 97 has invalid length (0 or 17)", "mac", pkt.ClientHWAddr.String(), "option 97", string(guid)) - err = fmt.Errorf("%w: option 97 has invalid length (0 or 17)", err) - } - - return err -} - // encodeToAttributes takes a DHCP packet and returns opentelemetry key/value attributes. func (h *Handler) encodeToAttributes(d *dhcpv4.DHCPv4, namespace string) []attribute.KeyValue { h.setDefaults() diff --git a/internal/dhcp/handler/reservation/handler_test.go b/internal/dhcp/handler/reservation/handler_test.go index e5378f58..77fd7ed8 100644 --- a/internal/dhcp/handler/reservation/handler_test.go +++ b/internal/dhcp/handler/reservation/handler_test.go @@ -12,7 +12,6 @@ import ( "testing" "time" - "github.com/go-logr/logr" "github.com/go-logr/stdr" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" @@ -517,58 +516,6 @@ func TestReadBackend(t *testing.T) { } } -func TestIsNetbootClient(t *testing.T) { - tests := map[string]struct { - input *dhcpv4.DHCPv4 - want error - }{ - "fail invalid message type": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptMessageType(dhcpv4.MessageTypeInform))}, want: errors.New("")}, - "fail no opt60": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover))}, want: errors.New("")}, - "fail bad opt60": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( - dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), - dhcpv4.OptClassIdentifier("BadClient"), - )}, want: errors.New("")}, - "fail no opt93": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( - dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), - dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), - )}, want: errors.New("")}, - "fail no opt94": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( - dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), - dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), - dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), - )}, want: errors.New("")}, - "fail invalid opt97[0] != 0": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( - dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), - dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), - dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), - dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), - dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05, 0x06, 0x00, 0x02, 0x03, 0x04, 0x05}), - )}, want: errors.New("")}, - "fail invalid len(opt97)": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( - dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), - dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), - dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), - dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), - dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{0x01, 0x02}), - )}, want: errors.New("")}, - "success len(opt97) == 0": {input: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList( - dhcpv4.OptMessageType(dhcpv4.MessageTypeDiscover), - dhcpv4.OptClassIdentifier("HTTPClient:Arch:xxxxx:UNDI:yyyzzz"), - dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), - dhcpv4.OptGeneric(dhcpv4.OptionClientNetworkInterfaceIdentifier, []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}), - dhcpv4.OptGeneric(dhcpv4.OptionClientMachineIdentifier, []byte{}), - )}, want: nil}, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - s := &Handler{Log: logr.Discard()} - if err := s.isNetbootClient(tt.input); (err == nil) != (tt.want == nil) { - t.Errorf("isNetbootClient() = %v, want %v", err, tt.want) - } - }) - } -} - func TestEncodeToAttributes(t *testing.T) { tests := map[string]struct { input *dhcpv4.DHCPv4 diff --git a/internal/dhcp/handler/reservation/option.go b/internal/dhcp/handler/reservation/option.go index d14c9f48..37403872 100644 --- a/internal/dhcp/handler/reservation/option.go +++ b/internal/dhcp/handler/reservation/option.go @@ -10,68 +10,11 @@ import ( "github.com/equinix-labs/otel-init-go/otelhelpers" "github.com/insomniacslk/dhcp/dhcpv4" - "github.com/insomniacslk/dhcp/iana" + "github.com/tinkerbell/smee/internal/dhcp" "github.com/tinkerbell/smee/internal/dhcp/data" "github.com/tinkerbell/smee/internal/dhcp/otel" ) -// UserClass is DHCP option 77 (https://www.rfc-editor.org/rfc/rfc3004.html). -type UserClass string - -// clientType is from DHCP option 60. Normally only PXEClient or HTTPClient. -type clientType string - -const ( - pxeClient clientType = "PXEClient" - httpClient clientType = "HTTPClient" -) - -// known user-class types. must correspond to DHCP option 77 - User-Class -// https://www.rfc-editor.org/rfc/rfc3004.html -const ( - // If the client has had iPXE burned into its ROM (or is a VM - // that uses iPXE as the PXE "ROM"), special handling is - // needed because in this mode the client is using iPXE native - // drivers and chainloading to a UNDI stack won't work. - IPXE UserClass = "iPXE" - // If the client identifies as "Tinkerbell", we've already - // chainloaded this client to the full-featured copy of iPXE - // we supply. We have to distinguish this case so we don't - // loop on the chainload step. - Tinkerbell UserClass = "Tinkerbell" -) - -// ArchToBootFile maps supported hardware PXE architectures types to iPXE binary files. -var ArchToBootFile = map[iana.Arch]string{ - iana.INTEL_X86PC: "undionly.kpxe", - iana.NEC_PC98: "undionly.kpxe", - iana.EFI_ITANIUM: "undionly.kpxe", - iana.DEC_ALPHA: "undionly.kpxe", - iana.ARC_X86: "undionly.kpxe", - iana.INTEL_LEAN_CLIENT: "undionly.kpxe", - iana.EFI_IA32: "ipxe.efi", - iana.EFI_X86_64: "ipxe.efi", - iana.EFI_XSCALE: "ipxe.efi", - iana.EFI_BC: "ipxe.efi", - iana.EFI_ARM32: "snp.efi", - iana.EFI_ARM64: "snp.efi", - iana.EFI_X86_HTTP: "ipxe.efi", - iana.EFI_X86_64_HTTP: "ipxe.efi", - iana.EFI_ARM32_HTTP: "snp.efi", - iana.EFI_ARM64_HTTP: "snp.efi", - iana.Arch(41): "snp.efi", // arm rpiboot: https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture -} - -// String function for clientType. -func (c clientType) String() string { - return string(c) -} - -// String function for UserClass. -func (u UserClass) String() string { - return string(u) -} - // setDHCPOpts takes a client dhcp packet and data (typically from a backend) and creates a slice of DHCP packet modifiers. // m is the DHCP request from a client. d is the data to use to create the DHCP packet modifiers. // This is most likely the place where we would have any business logic for determining DHCP option setting. @@ -126,24 +69,20 @@ func (h *Handler) setNetworkBootOpts(ctx context.Context, m *dhcpv4.DHCPv4, n *d // m is a received DHCPv4 packet. // d is the reply packet we are building. withNetboot := func(d *dhcpv4.DHCPv4) { - var opt60 string // if the client sends opt 60 with HTTPClient then we need to respond with opt 60 if val := m.Options.Get(dhcpv4.OptionClassIdentifier); val != nil { - if strings.HasPrefix(string(val), httpClient.String()) { - d.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClassIdentifier, []byte(httpClient))) - opt60 = httpClient.String() + if strings.HasPrefix(string(val), dhcp.HTTPClient.String()) { + d.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClassIdentifier, []byte(dhcp.HTTPClient))) } } d.BootFileName = "/netboot-not-allowed" d.ServerIPAddr = net.IPv4(0, 0, 0, 0) if n.AllowNetboot { - a := arch(m) - bin, found := ArchToBootFile[a] - if !found { - h.Log.Error(fmt.Errorf("unable to find bootfile for arch"), "network boot not allowed", "arch", a, "archInt", int(a), "mac", m.ClientHWAddr) + i := dhcp.NewInfo(m) + if i.IPXEBinary == "" { return } - uClass := UserClass(string(m.GetOneOption(dhcpv4.OptionUserClassInformation))) + uClass := dhcp.UserClass(string(m.GetOneOption(dhcpv4.OptionUserClassInformation))) var ipxeScript *url.URL if h.Netboot.IPXEScriptURL != nil { ipxeScript = h.Netboot.IPXEScriptURL(m) @@ -151,7 +90,7 @@ func (h *Handler) setNetworkBootOpts(ctx context.Context, m *dhcpv4.DHCPv4, n *d if n.IPXEScriptURL != nil { ipxeScript = n.IPXEScriptURL } - d.BootFileName, d.ServerIPAddr = h.bootfileAndNextServer(ctx, uClass, opt60, bin, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, ipxeScript) + d.BootFileName, d.ServerIPAddr = h.bootfileAndNextServer(ctx, m, uClass, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, ipxeScript) pxe := dhcpv4.Options{ // FYI, these are suboptions of option43. ref: https://datatracker.ietf.org/doc/html/rfc2132#section-8.4 // PXE Boot Server Discovery Control - bypass, just boot from filename. 6: []byte{8}, @@ -167,60 +106,15 @@ func (h *Handler) setNetworkBootOpts(ctx context.Context, m *dhcpv4.DHCPv4, n *d // bootfileAndNextServer returns the bootfile (string) and next server (net.IP). // input arguments `tftp`, `ipxe` and `iscript` use non string types so as to attempt to be more clear about the expectation around what is wanted for these values. // It also helps us avoid having to validate a string in multiple ways. -func (h *Handler) bootfileAndNextServer(ctx context.Context, uClass UserClass, opt60, bin string, tftp netip.AddrPort, ipxe, iscript *url.URL) (string, net.IP) { +func (h *Handler) bootfileAndNextServer(ctx context.Context, pkt *dhcpv4.DHCPv4, uClass dhcp.UserClass, tftp netip.AddrPort, ipxe, iscript *url.URL) (string, net.IP) { var nextServer net.IP var bootfile string + i := dhcp.NewInfo(pkt) if tp := otelhelpers.TraceparentStringFromContext(ctx); h.OTELEnabled && tp != "" { - bin = fmt.Sprintf("%s-%v", bin, tp) - } - // If a machine is in an ipxe boot loop, it is likely to be that we aren't matching on IPXE or Tinkerbell userclass (option 77). - switch { // order matters here. - case uClass == Tinkerbell, (h.Netboot.UserClass != "" && uClass == h.Netboot.UserClass): // this case gets us out of an ipxe boot loop. - bootfile = "/no-ipxe-script-defined" - if iscript != nil { - bootfile = iscript.String() - } - case clientType(opt60) == httpClient: // Check the client type from option 60. - bootfile = ipxe.JoinPath(bin).String() - nextServer = net.ParseIP("0.0.0.0") - if n, err := netip.ParseAddrPort(ipxe.Host); err == nil { - nextServer = n.Addr().AsSlice() - } else if n2 := net.ParseIP(ipxe.Host); n2 != nil { - nextServer = net.ParseIP(ipxe.Host) - } - case uClass == IPXE: // if the "iPXE" user class is found it means we aren't in our custom version of ipxe, but because of the option 43 we're setting we need to give a full tftp url from which to boot. - bootfile = fmt.Sprintf("tftp://%v/%v", tftp.String(), bin) - nextServer = net.IP(tftp.Addr().AsSlice()) - default: - bootfile = bin - nextServer = net.IP(tftp.Addr().AsSlice()) + i.IPXEBinary = fmt.Sprintf("%s-%v", i.IPXEBinary, tp) } + nextServer = i.NextServer(ipxe, tftp) + bootfile = i.Bootfile(uClass, iscript, ipxe, tftp) return bootfile, nextServer } - -// arch returns the arch of the client pulled from DHCP option 93. -func arch(d *dhcpv4.DHCPv4) iana.Arch { - // get option 93 ; arch - fwt := d.ClientArch() - if len(fwt) == 0 { - return iana.Arch(255) // unknown arch - } - var archKnown bool - var a iana.Arch - for _, elem := range fwt { - if !strings.Contains(elem.String(), "unknown") { - archKnown = true - // Basic architecture identification, based purely on - // the PXE architecture option. - // https://www.iana.org/assignments/dhcpv6-parameters/dhcpv6-parameters.xhtml#processor-architecture - a = elem - break - } - } - if !archKnown { - return iana.Arch(255) // unknown arch - } - - return a -} diff --git a/internal/dhcp/handler/reservation/option_test.go b/internal/dhcp/handler/reservation/option_test.go index 65057350..b5ce62b6 100644 --- a/internal/dhcp/handler/reservation/option_test.go +++ b/internal/dhcp/handler/reservation/option_test.go @@ -15,12 +15,18 @@ import ( "github.com/insomniacslk/dhcp/dhcpv4" "github.com/insomniacslk/dhcp/iana" "github.com/insomniacslk/dhcp/rfc1035label" + "github.com/tinkerbell/smee/internal/dhcp" "github.com/tinkerbell/smee/internal/dhcp/data" oteldhcp "github.com/tinkerbell/smee/internal/dhcp/otel" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/propagation" ) +const ( + examplePXEClient = "PXEClient:Arch:00007:UNDI:003001" + exampleHTTPClient = "HTTPClient:Arch:00016:UNDI:003001" +) + func TestSetDHCPOpts(t *testing.T) { type args struct { in0 context.Context @@ -117,40 +123,10 @@ func TestSetDHCPOpts(t *testing.T) { } } -func TestArch(t *testing.T) { - tests := map[string]struct { - pkt *dhcpv4.DHCPv4 - want iana.Arch - }{ - "found": { - pkt: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptClientArch(iana.INTEL_X86PC))}, - want: iana.INTEL_X86PC, - }, - "unknown": { - pkt: &dhcpv4.DHCPv4{Options: dhcpv4.OptionsFromList(dhcpv4.OptClientArch(iana.Arch(255)))}, - want: iana.Arch(255), - }, - "unknown: opt 93 len 0": { - pkt: &dhcpv4.DHCPv4{}, - want: iana.Arch(255), - }, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - got := arch(tt.pkt) - if diff := cmp.Diff(tt.want, got); diff != "" { - t.Fatal(diff) - } - }) - } -} - func TestBootfileAndNextServer(t *testing.T) { type args struct { - mac net.HardwareAddr - uClass UserClass - opt60 string - bin string + pkt *dhcpv4.DHCPv4 + uClass dhcp.UserClass tftp netip.AddrPort ipxe *url.URL iscript *url.URL @@ -165,7 +141,12 @@ func TestBootfileAndNextServer(t *testing.T) { "success bootfile only": { server: &Handler{Log: logr.Discard()}, args: args{ - uClass: Tinkerbell, + pkt: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptUserClass(dhcp.Tinkerbell.String()), + ), + }, iscript: &url.URL{Scheme: "http", Host: "localhost:8080", Path: "/auto.ipxe"}, }, wantBootFile: "http://localhost:8080/auto.ipxe", @@ -174,59 +155,78 @@ func TestBootfileAndNextServer(t *testing.T) { "success httpClient": { server: &Handler{Log: logr.Discard()}, args: args{ - mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, - opt60: httpClient.String(), - bin: "snp.ipxe", - ipxe: &url.URL{Scheme: "http", Host: "localhost:8181"}, + pkt: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptClientArch(iana.EFI_ARM64_HTTP), + dhcpv4.OptClassIdentifier(exampleHTTPClient), + ), + }, + ipxe: &url.URL{Scheme: "http", Host: "127.0.0.1:8181"}, }, - wantBootFile: "http://localhost:8181/snp.ipxe", - wantNextSrv: net.IPv4(0, 0, 0, 0), + wantBootFile: "http://127.0.0.1:8181/01:02:03:04:05:06/snp.efi", + wantNextSrv: net.IPv4(127, 0, 0, 1), }, "success userclass iPXE": { server: &Handler{Log: logr.Discard()}, args: args{ - mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, - uClass: IPXE, - bin: "unidonly.kpxe", - tftp: netip.MustParseAddrPort("192.168.6.5:69"), - ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, + pkt: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptClientArch(iana.INTEL_X86PC), + dhcpv4.OptUserClass(dhcp.IPXE.String()), + ), + }, + tftp: netip.MustParseAddrPort("192.168.6.5:69"), }, - wantBootFile: "tftp://192.168.6.5:69/unidonly.kpxe", + wantBootFile: "tftp://192.168.6.5:69/01:02:03:04:05:07/undionly.kpxe", wantNextSrv: net.ParseIP("192.168.6.5"), }, "success userclass iPXE with otel": { server: &Handler{Log: logr.Discard(), OTELEnabled: true}, otelEnabled: true, args: args{ - mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, - uClass: IPXE, - bin: "unidonly.kpxe", - tftp: netip.MustParseAddrPort("192.168.6.5:69"), - ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, + pkt: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptClientArch(iana.INTEL_X86PC), + dhcpv4.OptUserClass(dhcp.IPXE.String()), + ), + }, + tftp: netip.MustParseAddrPort("192.168.6.5:69"), + ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, }, - wantBootFile: "tftp://192.168.6.5:69/unidonly.kpxe-00-23b1e307bb35484f535a1f772c06910e-d887dc3912240434-01", + wantBootFile: "tftp://192.168.6.5:69/01:02:03:04:05:07/undionly.kpxe-00-23b1e307bb35484f535a1f772c06910e-d887dc3912240434-01", wantNextSrv: net.ParseIP("192.168.6.5"), }, "success default": { server: &Handler{Log: logr.Discard()}, args: args{ - mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, - bin: "unidonly.kpxe", + pkt: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptClientArch(iana.INTEL_X86PC), + ), + }, tftp: netip.MustParseAddrPort("192.168.6.5:69"), ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, }, - wantBootFile: "unidonly.kpxe", + wantBootFile: "undionly.kpxe", wantNextSrv: net.ParseIP("192.168.6.5"), }, "success otel enabled, no traceparent": { server: &Handler{Log: logr.Discard(), OTELEnabled: true}, args: args{ - mac: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, - bin: "unidonly.kpxe", + pkt: &dhcpv4.DHCPv4{ + ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x07}, + Options: dhcpv4.OptionsFromList( + dhcpv4.OptClientArch(iana.INTEL_X86PC), + ), + }, tftp: netip.MustParseAddrPort("192.168.6.5:69"), ipxe: &url.URL{Scheme: "tftp", Host: "192.168.6.5:69"}, }, - wantBootFile: "unidonly.kpxe", + wantBootFile: "undionly.kpxe", wantNextSrv: net.ParseIP("192.168.6.5"), }, } @@ -239,11 +239,11 @@ func TestBootfileAndNextServer(t *testing.T) { otel.SetTextMapPropagator(prop) ctx = otelhelpers.ContextWithTraceparentString(ctx, "00-23b1e307bb35484f535a1f772c06910e-d887dc3912240434-01") } - bootfile, nextServer := tt.server.bootfileAndNextServer(ctx, tt.args.uClass, tt.args.opt60, tt.args.bin, tt.args.tftp, tt.args.ipxe, tt.args.iscript) - if diff := cmp.Diff(tt.wantBootFile, bootfile); diff != "" { + bootfile, nextServer := tt.server.bootfileAndNextServer(ctx, tt.args.pkt, tt.args.uClass, tt.args.tftp, tt.args.ipxe, tt.args.iscript) + if diff := cmp.Diff(bootfile, tt.wantBootFile); diff != "" { t.Fatal("bootfile", diff) } - if diff := cmp.Diff(tt.wantNextSrv, nextServer); diff != "" { + if diff := cmp.Diff(nextServer, tt.wantNextSrv); diff != "" { t.Fatal("nextServer", diff) } }) @@ -279,7 +279,7 @@ func TestSetNetworkBootOpts(t *testing.T) { m: &dhcpv4.DHCPv4{ ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, Options: dhcpv4.OptionsFromList( - dhcpv4.OptUserClass(Tinkerbell.String()), + dhcpv4.OptUserClass(dhcp.Tinkerbell.String()), dhcpv4.OptClassIdentifier("HTTPClient:xxxxx"), dhcpv4.OptClientArch(iana.EFI_X86_64_HTTP), ), @@ -303,7 +303,7 @@ func TestSetNetworkBootOpts(t *testing.T) { m: &dhcpv4.DHCPv4{ ClientHWAddr: net.HardwareAddr{0x01, 0x02, 0x03, 0x04, 0x05, 0x06}, Options: dhcpv4.OptionsFromList( - dhcpv4.OptUserClass(Tinkerbell.String()), + dhcpv4.OptUserClass(dhcp.Tinkerbell.String()), dhcpv4.OptClientArch(iana.UBOOT_ARM64), ), }, @@ -329,7 +329,7 @@ func TestSetNetworkBootOpts(t *testing.T) { gotFunc := s.setNetworkBootOpts(tt.args.in0, tt.args.m, tt.args.n) got := new(dhcpv4.DHCPv4) gotFunc(got) - if diff := cmp.Diff(tt.want, got); diff != "" { + if diff := cmp.Diff(got, tt.want); diff != "" { t.Fatal(diff) } }) diff --git a/internal/dhcp/handler/reservation/reservation.go b/internal/dhcp/handler/reservation/reservation.go index 2d68a568..bca234b2 100644 --- a/internal/dhcp/handler/reservation/reservation.go +++ b/internal/dhcp/handler/reservation/reservation.go @@ -7,6 +7,7 @@ import ( "github.com/go-logr/logr" "github.com/insomniacslk/dhcp/dhcpv4" + "github.com/tinkerbell/smee/internal/dhcp" "github.com/tinkerbell/smee/internal/dhcp/handler" ) @@ -52,5 +53,5 @@ type Netboot struct { Enabled bool // UserClass (for network booting) allows a custom DHCP option 77 to be used to break out of an iPXE loop. - UserClass UserClass + UserClass dhcp.UserClass } From 5f81f01fc73907f4cfd05d3da2bb92728dcd8310 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Wed, 27 Dec 2023 17:40:35 -0700 Subject: [PATCH 05/17] Fix linting issues Signed-off-by: Jacob Weinstock --- docs/Modes.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/docs/Modes.md b/docs/Modes.md index 6c32eea4..85ffc4f4 100644 --- a/docs/Modes.md +++ b/docs/Modes.md @@ -6,8 +6,6 @@ Smee's DHCP functionality can operate in one of three modes: - **proxDHCP**: Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. -- **DHCP disabled**: Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network and you want to use Smee's TFTP and HTTP functionality. - - +- **DHCP disabled**: Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network and you want to use Smee's TFTP and HTTP functionality. In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the request to lookup the corresponding hardware. From c976e8f5ed7e4add45e5863982a8cad2fa4301ef Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Dec 2023 10:19:54 -0700 Subject: [PATCH 06/17] Some clean up of around the customer user class: The custom user class was not being set from the handler but from the request packet. This was not correct. Signed-off-by: Jacob Weinstock --- internal/dhcp/handler/reservation/option.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/dhcp/handler/reservation/option.go b/internal/dhcp/handler/reservation/option.go index 37403872..8e3d22d1 100644 --- a/internal/dhcp/handler/reservation/option.go +++ b/internal/dhcp/handler/reservation/option.go @@ -70,6 +70,7 @@ func (h *Handler) setNetworkBootOpts(ctx context.Context, m *dhcpv4.DHCPv4, n *d // d is the reply packet we are building. withNetboot := func(d *dhcpv4.DHCPv4) { // if the client sends opt 60 with HTTPClient then we need to respond with opt 60 + // This is outside of the n.AllowNetboot check because we will be sending "/netboot-not-allowed" regardless. if val := m.Options.Get(dhcpv4.OptionClassIdentifier); val != nil { if strings.HasPrefix(string(val), dhcp.HTTPClient.String()) { d.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionClassIdentifier, []byte(dhcp.HTTPClient))) @@ -82,21 +83,22 @@ func (h *Handler) setNetworkBootOpts(ctx context.Context, m *dhcpv4.DHCPv4, n *d if i.IPXEBinary == "" { return } - uClass := dhcp.UserClass(string(m.GetOneOption(dhcpv4.OptionUserClassInformation))) var ipxeScript *url.URL + // If the global IPXEScriptURL is set, use that. if h.Netboot.IPXEScriptURL != nil { ipxeScript = h.Netboot.IPXEScriptURL(m) } + // If the IPXE script URL is set on the hardware record, use that. if n.IPXEScriptURL != nil { ipxeScript = n.IPXEScriptURL } - d.BootFileName, d.ServerIPAddr = h.bootfileAndNextServer(ctx, m, uClass, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, ipxeScript) + d.BootFileName, d.ServerIPAddr = h.bootfileAndNextServer(ctx, m, h.Netboot.UserClass, h.Netboot.IPXEBinServerTFTP, h.Netboot.IPXEBinServerHTTP, ipxeScript) pxe := dhcpv4.Options{ // FYI, these are suboptions of option43. ref: https://datatracker.ietf.org/doc/html/rfc2132#section-8.4 // PXE Boot Server Discovery Control - bypass, just boot from filename. 6: []byte{8}, 69: otel.TraceparentFromContext(ctx), } - d.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, pxe.ToBytes())) + d.UpdateOption(dhcpv4.OptGeneric(dhcpv4.OptionVendorSpecificInformation, i.AddRPIOpt43(pxe))) } } @@ -106,7 +108,7 @@ func (h *Handler) setNetworkBootOpts(ctx context.Context, m *dhcpv4.DHCPv4, n *d // bootfileAndNextServer returns the bootfile (string) and next server (net.IP). // input arguments `tftp`, `ipxe` and `iscript` use non string types so as to attempt to be more clear about the expectation around what is wanted for these values. // It also helps us avoid having to validate a string in multiple ways. -func (h *Handler) bootfileAndNextServer(ctx context.Context, pkt *dhcpv4.DHCPv4, uClass dhcp.UserClass, tftp netip.AddrPort, ipxe, iscript *url.URL) (string, net.IP) { +func (h *Handler) bootfileAndNextServer(ctx context.Context, pkt *dhcpv4.DHCPv4, customUC dhcp.UserClass, tftp netip.AddrPort, ipxe, iscript *url.URL) (string, net.IP) { var nextServer net.IP var bootfile string i := dhcp.NewInfo(pkt) @@ -114,7 +116,7 @@ func (h *Handler) bootfileAndNextServer(ctx context.Context, pkt *dhcpv4.DHCPv4, i.IPXEBinary = fmt.Sprintf("%s-%v", i.IPXEBinary, tp) } nextServer = i.NextServer(ipxe, tftp) - bootfile = i.Bootfile(uClass, iscript, ipxe, tftp) + bootfile = i.Bootfile(customUC, iscript, ipxe, tftp) return bootfile, nextServer } From d3919ccc5ede1c8f5b22ecc94cf8ac1e4d5e767f Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Dec 2023 12:19:24 -0700 Subject: [PATCH 07/17] Add otel spans, code comments: Signed-off-by: Jacob Weinstock --- internal/dhcp/dhcp.go | 21 +++++++++++++++------ internal/dhcp/handler/proxy/proxy.go | 22 +++++++++++++++++++++- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/internal/dhcp/dhcp.go b/internal/dhcp/dhcp.go index d1aafc75..ba9258b5 100644 --- a/internal/dhcp/dhcp.go +++ b/internal/dhcp/dhcp.go @@ -62,13 +62,22 @@ var ArchToBootFile = map[iana.Arch]string{ // ErrUnknownArch is used when the PXE client request is from an unknown architecture. var ErrUnknownArch = fmt.Errorf("could not determine client architecture from option 93") +// Info holds details about the dhcp request. Use NewInfo to populate the struct fields from a dhcp packet. type Info struct { - Pkt *dhcpv4.DHCPv4 - Arch iana.Arch - Mac net.HardwareAddr - UserClass UserClass - ClientType ClientType + // Pkt is the dhcp packet that was received from the client. + Pkt *dhcpv4.DHCPv4 + // Arch is the architecture of the client. Use NewInfo to automatically populate this field. + Arch iana.Arch + // Mac is the mac address of the client. Use NewInfo to automatically populate this field. + Mac net.HardwareAddr + // UserClass is the user class of the client. Use NewInfo to automatically populate this field. + UserClass UserClass + // ClientType is the client type of the client. Use NewInfo to automatically populate this field. + ClientType ClientType + // IsNetbootClient returns nil if the client is a valid netboot client. Otherwise it returns an error. + // Use NewInfo to automatically populate this field. IsNetbootClient error + // IPXEBinary is the iPXE binary file to boot. Use NewInfo to automatically populate this field. IPXEBinary string } @@ -157,7 +166,7 @@ func (i Info) ClientTypeFrom() ClientType { return c } -// IsNetbootClient returns true if the client is a valid netboot client. +// IsNetbootClient returns nil if the client is a valid netboot client. Otherwise it returns an error. // // A valid netboot client will have the following in its DHCP request: // 1. is a DHCP discovery/request message type. diff --git a/internal/dhcp/handler/proxy/proxy.go b/internal/dhcp/handler/proxy/proxy.go index 608d98bf..0de6f559 100644 --- a/internal/dhcp/handler/proxy/proxy.go +++ b/internal/dhcp/handler/proxy/proxy.go @@ -29,6 +29,7 @@ import ( oteldhcp "github.com/tinkerbell/smee/internal/dhcp/otel" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" "go.opentelemetry.io/otel/trace" "golang.org/x/net/ipv4" ) @@ -122,11 +123,15 @@ func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, dp data.Pac if dp.Pkt.OpCode != dhcpv4.OpcodeBootRequest { // TODO(jacobweinstock): dont understand this, found it in an example here: https://github.com/insomniacslk/dhcp/blob/c51060810aaab9c8a0bd1b0fcbf72bc0b91e6427/dhcpv4/server4/server_test.go#L31 log.V(1).Info("Ignoring packet", "OpCode", dp.Pkt.OpCode) + span.SetStatus(codes.Ok, "Ignoring packet: OpCode not BootRequest") + return } if err := setMessageType(reply, dp.Pkt.MessageType()); err != nil { log.V(1).Info("Ignoring packet", "error", err.Error()) + span.SetStatus(codes.Ok, err.Error()) + return } @@ -135,12 +140,22 @@ func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, dp data.Pac i := dhcp.NewInfo(dp.Pkt) + if !h.Netboot.Enabled { + log.V(1).Info("Ignoring packet: netboot is not enabled") + span.SetStatus(codes.Ok, "Ignoring packet: netboot is not enabled") + + return + } if err := i.IsNetbootClient; err != nil { log.V(1).Info("Ignoring packet: not from a PXE enabled client", "error", err.Error()) + span.SetStatus(codes.Ok, fmt.Sprintf("Ignoring packet: not from a PXE enabled client: %s", err.Error())) + return } if i.IPXEBinary == "" { log.V(1).Info("Ignoring packet: no iPXE binary was able to be determined") + span.SetStatus(codes.Ok, "Ignoring packet: no iPXE binary was able to be determined") + return } @@ -172,7 +187,8 @@ func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, dp data.Pac // check the backend, if PXE is NOT allowed, set the boot file name to "//not-allowed" _, n, err := h.Backend.GetByMac(ctx, dp.Pkt.ClientHWAddr) if err != nil || (n != nil && !n.AllowNetboot) { - log.V(1).Info("Ignoring packet", "error", err.Error(), "netbootAllowed", n) + log.V(1).Info("Ignoring packet", "error", err.Error(), "netbootAllowed", n.AllowNetboot) + span.SetStatus(codes.Ok, "netboot not allowed") return } log.Info( @@ -197,9 +213,13 @@ func (h *Handler) Handle(ctx context.Context, conn *ipv4.PacketConn, dp data.Pac // send the DHCP packet if _, err := conn.WriteTo(reply.ToBytes(), cm, dst); err != nil { log.Error(err, "failed to send ProxyDHCP response") + span.SetStatus(codes.Error, err.Error()) + return } log.Info("Sent ProxyDHCP response") + span.SetAttributes(h.encodeToAttributes(reply, "reply")...) + span.SetStatus(codes.Ok, "sent DHCP response") } // encodeToAttributes takes a DHCP packet and returns opentelemetry key/value attributes. From 606a52880e84f8c6508b59e0594f3759b097a8f5 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Dec 2023 12:20:06 -0700 Subject: [PATCH 08/17] Fix linting issues Signed-off-by: Jacob Weinstock --- internal/dhcp/dhcp.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/dhcp/dhcp.go b/internal/dhcp/dhcp.go index ba9258b5..9f73bd33 100644 --- a/internal/dhcp/dhcp.go +++ b/internal/dhcp/dhcp.go @@ -65,20 +65,20 @@ var ErrUnknownArch = fmt.Errorf("could not determine client architecture from op // Info holds details about the dhcp request. Use NewInfo to populate the struct fields from a dhcp packet. type Info struct { // Pkt is the dhcp packet that was received from the client. - Pkt *dhcpv4.DHCPv4 + Pkt *dhcpv4.DHCPv4 // Arch is the architecture of the client. Use NewInfo to automatically populate this field. - Arch iana.Arch + Arch iana.Arch // Mac is the mac address of the client. Use NewInfo to automatically populate this field. - Mac net.HardwareAddr + Mac net.HardwareAddr // UserClass is the user class of the client. Use NewInfo to automatically populate this field. - UserClass UserClass + UserClass UserClass // ClientType is the client type of the client. Use NewInfo to automatically populate this field. ClientType ClientType // IsNetbootClient returns nil if the client is a valid netboot client. Otherwise it returns an error. // Use NewInfo to automatically populate this field. IsNetbootClient error // IPXEBinary is the iPXE binary file to boot. Use NewInfo to automatically populate this field. - IPXEBinary string + IPXEBinary string } func NewInfo(pkt *dhcpv4.DHCPv4) Info { From 1fe9ccb74e4c4c2ab3dac6161cfc35f5e9eb1d79 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Dec 2023 15:36:38 -0700 Subject: [PATCH 09/17] Update Doc on DHCP modes Signed-off-by: Jacob Weinstock --- docs/Modes.md | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/docs/Modes.md b/docs/Modes.md index 85ffc4f4..cc864559 100644 --- a/docs/Modes.md +++ b/docs/Modes.md @@ -1,11 +1,15 @@ # DHCP Modes -Smee's DHCP functionality can operate in one of three modes: +Smee's DHCP functionality can operate in one of the following modes: -- **DHCP Reservation**: Smee will respond to DHCP requests from clients and provide them with IP and next boot info. This is the default mode. IP info is all reservation based. There must be a corresponding hardware record for the requesting client's MAC address. +1. **DHCP Reservation** +Smee will respond to DHCP requests from clients and provide them with IP and next boot info when netbooting. This is the default mode. IP info is all reservation based. There must be a corresponding hardware record for the requesting client's MAC address. +To enable this mode set `-dhcp-mode=reservation`. -- **proxDHCP**: Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. +1. **Proxy DHCP** +Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info when netbooting. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. +To enable this mode set `-dhcp-mode=proxy`. -- **DHCP disabled**: Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network and you want to use Smee's TFTP and HTTP functionality. - -In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the request to lookup the corresponding hardware. +1. **DHCP disabled** +Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network that will provide both IP and next boot info and you want to use Smee's TFTP and HTTP functionality. In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the client request to lookup the corresponding hardware. There must be a corresponding hardware record for the requesting client's IP address. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. +To enable this mode set `-dhcp-mode=disabled`. From 9c37ab2a0cfd270bf2ef49d7a41b3ff0853a94ee Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Dec 2023 16:14:04 -0700 Subject: [PATCH 10/17] CLI flags and implementation for dhcp mode: DHCP reservation and proxy can now be toggled between via cli flag/env var. Signed-off-by: Jacob Weinstock --- cmd/smee/flag.go | 1 + cmd/smee/flag_test.go | 2 ++ cmd/smee/main.go | 80 +++++++++++++++++++++++-------------------- docs/Modes.md | 2 +- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/cmd/smee/flag.go b/cmd/smee/flag.go index 5cad6764..14ee343c 100644 --- a/cmd/smee/flag.go +++ b/cmd/smee/flag.go @@ -110,6 +110,7 @@ func ipxeHTTPScriptFlags(c *config, fs *flag.FlagSet) { func dhcpFlags(c *config, fs *flag.FlagSet) { fs.BoolVar(&c.dhcp.enabled, "dhcp-enabled", true, "[dhcp] enable DHCP server") + fs.StringVar(&c.dhcp.mode, "dhcp-mode", "reservation", "[dhcp] DHCP mode (reservation, proxy)") fs.StringVar(&c.dhcp.bindAddr, "dhcp-addr", "0.0.0.0:67", "[dhcp] local IP:Port to listen on for DHCP requests") fs.StringVar(&c.dhcp.bindInterface, "dhcp-iface", "", "[dhcp] interface to bind to for DHCP requests") fs.StringVar(&c.dhcp.ipForPacket, "dhcp-ip-for-packet", detectPublicIPv4(""), "[dhcp] IP address to use in DHCP packets (opt 54, etc)") diff --git a/cmd/smee/flag_test.go b/cmd/smee/flag_test.go index 8b9e6be5..c8d44e1c 100644 --- a/cmd/smee/flag_test.go +++ b/cmd/smee/flag_test.go @@ -30,6 +30,7 @@ func TestParser(t *testing.T) { }, dhcp: dhcpConfig{ enabled: true, + mode: "reservation", bindAddr: "0.0.0.0:67", ipForPacket: "192.168.2.4", syslogIP: "192.168.2.4", @@ -98,6 +99,7 @@ FLAGS -dhcp-http-ipxe-script-url [dhcp] HTTP iPXE script URL to use in DHCP packets (default "http://%[1]v/auto.ipxe") -dhcp-iface [dhcp] interface to bind to for DHCP requests -dhcp-ip-for-packet [dhcp] IP address to use in DHCP packets (opt 54, etc) (default "%[1]v") + -dhcp-mode [dhcp] DHCP mode (reservation, proxy) (default "reservation") -dhcp-syslog-ip [dhcp] Syslog server IP address to use in DHCP packets (opt 7) (default "%[1]v") -dhcp-tftp-ip [dhcp] TFTP server IP address to use in DHCP packets (opt 66, etc) (default "%[1]v:69") -disable-discover-trusted-proxies [http] disable discovery of trusted proxies from Kubernetes, only available for the Kubernetes backend (default "false") diff --git a/cmd/smee/main.go b/cmd/smee/main.go index b90bf3d0..9f9d7895 100644 --- a/cmd/smee/main.go +++ b/cmd/smee/main.go @@ -24,6 +24,7 @@ import ( "github.com/tinkerbell/ipxedust/ihttp" "github.com/tinkerbell/smee/internal/dhcp/handler" "github.com/tinkerbell/smee/internal/dhcp/handler/proxy" + "github.com/tinkerbell/smee/internal/dhcp/handler/reservation" "github.com/tinkerbell/smee/internal/dhcp/server" "github.com/tinkerbell/smee/internal/ipxe/http" "github.com/tinkerbell/smee/internal/ipxe/script" @@ -85,6 +86,7 @@ type ipxeHTTPScript struct { type dhcpConfig struct { enabled bool + mode string bindAddr string bindInterface string ipForPacket string @@ -221,7 +223,7 @@ func main() { }) } - // dhcp server + // dhcp serving if cfg.dhcp.enabled { dh, err := cfg.dhcpHandler(ctx, log) if err != nil { @@ -252,8 +254,7 @@ func main() { log.Info("smee is shutting down") } -// func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*reservation.Handler, error) {. -func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*proxy.Handler, error) { +func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (server.Handler, error) { // 1. create the handler // 2. create the backend // 3. add the backend to the handler @@ -284,37 +285,7 @@ func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*proxy.Handl return &u } } - syslogIP, err := netip.ParseAddr(c.dhcp.syslogIP) - if err != nil { - return nil, fmt.Errorf("invalid syslog address: %w", err) - } - log.V(19).Info("debug", "syslog", syslogIP) - dh := &proxy.Handler{ - Backend: nil, - IPAddr: pktIP, - Log: log, - Netboot: proxy.Netboot{ - IPXEBinServerTFTP: tftpIP, - IPXEBinServerHTTP: httpBinaryURL, - IPXEScriptURL: ipxeScript, - Enabled: true, - }, - OTELEnabled: true, - } - /*dh := &reservation.Handler{ - Backend: nil, - IPAddr: pktIP, - Log: log, - Netboot: reservation.Netboot{ - IPXEBinServerTFTP: tftpIP, - IPXEBinServerHTTP: httpBinaryURL, - IPXEScriptURL: ipxeScript, - Enabled: true, - }, - OTELEnabled: true, - SyslogAddr: syslogIP, - } - */ + var backend handler.BackendReader switch { case c.backends.file.Enabled && c.backends.kubernetes.Enabled: panic("only one backend can be enabled at a time") @@ -323,16 +294,51 @@ func (c *config) dhcpHandler(ctx context.Context, log logr.Logger) (*proxy.Handl if err != nil { return nil, fmt.Errorf("failed to create file backend: %w", err) } - dh.Backend = b + backend = b default: // default backend is kubernetes b, err := c.backends.kubernetes.backend(ctx) if err != nil { return nil, fmt.Errorf("failed to create kubernetes backend: %w", err) } - dh.Backend = b + backend = b + } + switch c.dhcp.mode { + case "reservation": + syslogIP, err := netip.ParseAddr(c.dhcp.syslogIP) + if err != nil { + return nil, fmt.Errorf("invalid syslog address: %w", err) + } + dh := &reservation.Handler{ + Backend: backend, + IPAddr: pktIP, + Log: log, + Netboot: reservation.Netboot{ + IPXEBinServerTFTP: tftpIP, + IPXEBinServerHTTP: httpBinaryURL, + IPXEScriptURL: ipxeScript, + Enabled: true, + }, + OTELEnabled: true, + SyslogAddr: syslogIP, + } + return dh, nil + case "proxy": + dh := &proxy.Handler{ + Backend: backend, + IPAddr: pktIP, + Log: log, + Netboot: proxy.Netboot{ + IPXEBinServerTFTP: tftpIP, + IPXEBinServerHTTP: httpBinaryURL, + IPXEScriptURL: ipxeScript, + Enabled: true, + }, + OTELEnabled: true, + } + return dh, nil } - return dh, nil + return nil, errors.New("invalid dhcp mode") } // defaultLogger is zap logr implementation. diff --git a/docs/Modes.md b/docs/Modes.md index cc864559..b3b83880 100644 --- a/docs/Modes.md +++ b/docs/Modes.md @@ -12,4 +12,4 @@ To enable this mode set `-dhcp-mode=proxy`. 1. **DHCP disabled** Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network that will provide both IP and next boot info and you want to use Smee's TFTP and HTTP functionality. In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the client request to lookup the corresponding hardware. There must be a corresponding hardware record for the requesting client's IP address. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. -To enable this mode set `-dhcp-mode=disabled`. +To enable this mode set `-dhcp-enabled=false`. From 5c1dcfc17a2e040b420d340f9746db85e7aa3292 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Dec 2023 16:15:11 -0700 Subject: [PATCH 11/17] Fix linting issues Signed-off-by: Jacob Weinstock --- docs/Modes.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/Modes.md b/docs/Modes.md index b3b83880..7b6c51a9 100644 --- a/docs/Modes.md +++ b/docs/Modes.md @@ -3,13 +3,13 @@ Smee's DHCP functionality can operate in one of the following modes: 1. **DHCP Reservation** -Smee will respond to DHCP requests from clients and provide them with IP and next boot info when netbooting. This is the default mode. IP info is all reservation based. There must be a corresponding hardware record for the requesting client's MAC address. -To enable this mode set `-dhcp-mode=reservation`. + Smee will respond to DHCP requests from clients and provide them with IP and next boot info when netbooting. This is the default mode. IP info is all reservation based. There must be a corresponding hardware record for the requesting client's MAC address. + To enable this mode set `-dhcp-mode=reservation`. 1. **Proxy DHCP** -Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info when netbooting. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. -To enable this mode set `-dhcp-mode=proxy`. + Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info when netbooting. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. + To enable this mode set `-dhcp-mode=proxy`. 1. **DHCP disabled** -Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network that will provide both IP and next boot info and you want to use Smee's TFTP and HTTP functionality. In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the client request to lookup the corresponding hardware. There must be a corresponding hardware record for the requesting client's IP address. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. -To enable this mode set `-dhcp-enabled=false`. + Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network that will provide both IP and next boot info and you want to use Smee's TFTP and HTTP functionality. In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the client request to lookup the corresponding hardware. There must be a corresponding hardware record for the requesting client's IP address. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. + To enable this mode set `-dhcp-enabled=false`. From 47efd2d4e4bc5eafa3108392b0b5755da0848747 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Thu, 28 Dec 2023 16:25:19 -0700 Subject: [PATCH 12/17] go mod tidy Signed-off-by: Jacob Weinstock --- go.sum | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index d183573d..52f25837 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0n github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= -github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -238,6 +238,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= From 7bae628da97501784fcbc1247a268c374f01d48d Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Mon, 15 Jan 2024 08:55:18 -0700 Subject: [PATCH 13/17] Go mod tidy Signed-off-by: Jacob Weinstock --- go.mod | 1 + go.sum | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 88164e16..b8598624 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/prometheus/client_golang v1.18.0 github.com/tinkerbell/ipxedust v0.0.0-20231215220341-a535c5deb47a github.com/tinkerbell/tink v0.9.0 + github.com/tonglil/buflogr v1.1.1 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 go.opentelemetry.io/otel v1.21.0 go.opentelemetry.io/otel/trace v1.21.0 diff --git a/go.sum b/go.sum index 52f25837..5219e01a 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,8 @@ github.com/evanphx/json-patch/v5 v5.7.0 h1:nJqP7uwL84RJInrohHfW0Fx3awjbm8qZeFv0n github.com/evanphx/json-patch/v5 v5.7.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= -github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= @@ -172,8 +172,8 @@ github.com/tinkerbell/ipxedust v0.0.0-20231215220341-a535c5deb47a h1:BNbuuQp8m/L github.com/tinkerbell/ipxedust v0.0.0-20231215220341-a535c5deb47a/go.mod h1:zrFXKJHUplvuggD9MzSQuZldQZU4CLter7QYqSLiiE4= github.com/tinkerbell/tink v0.9.0 h1:W7X/OEmhyYXE/kPVu1U31fpugVHoc2qsAvBtsZ7mkDg= github.com/tinkerbell/tink v0.9.0/go.mod h1:r8gDvx/Y+GEFeT9xwKa14ULrkMre8mYmH3/E9VbUkEw= -github.com/tonglil/buflogr v1.0.1 h1:WXFZLKxLfqcVSmckwiMCF8jJwjIgmStJmg63YKRF1p0= -github.com/tonglil/buflogr v1.0.1/go.mod h1:yYWwvSpn/3uAaqjf6mJg/XMiAciaR0QcRJH2gJGDxNE= +github.com/tonglil/buflogr v1.1.1 h1:CKAjOHBSMmgbRFxpn/RhQHPj5oANc7ekhlsoUDvcZIg= +github.com/tonglil/buflogr v1.1.1/go.mod h1:WLLtPRLqcFYWQLbA+ytXy5WrFTYnfA+beg1MpvJCxm4= github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63 h1:YcojQL98T/OO+rybuzn2+5KrD5dBwXIvYBvQ2cD3Avg= github.com/u-root/uio v0.0.0-20230305220412-3e8cd9d6bf63/go.mod h1:eLL9Nub3yfAho7qB0MzZizFhTU2QkLeoVsWdHtDW264= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -238,7 +238,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220622161953-175b2fd9d664/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= From fe75a3b042fd2395b55d4e8c5184d51dab550420 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Mon, 15 Jan 2024 09:01:03 -0700 Subject: [PATCH 14/17] Fix linting issues Signed-off-by: Jacob Weinstock --- backend/file/testdata/example.yaml | 100 ++++++++++++++--------------- dhcp/README.md | 6 +- dhcp/docs/Backend-File.md | 48 +++++++------- 3 files changed, 77 insertions(+), 77 deletions(-) diff --git a/backend/file/testdata/example.yaml b/backend/file/testdata/example.yaml index 0708ff05..6a4043a5 100644 --- a/backend/file/testdata/example.yaml +++ b/backend/file/testdata/example.yaml @@ -1,80 +1,80 @@ --- 08:00:27:29:4E:67: - ipAddress: '192.168.2.153' - subnetMask: '255.255.255.0' - defaultGateway: '192.168.2.1' + ipAddress: "192.168.2.153" + subnetMask: "255.255.255.0" + defaultGateway: "192.168.2.1" nameServers: - - '8.8.8.8' - - '1.1.1.1' - hostname: 'pxe-virtualbox' - domainName: 'example.com' - broadcastAddress: '192.168.2.255' + - "8.8.8.8" + - "1.1.1.1" + hostname: "pxe-virtualbox" + domainName: "example.com" + broadcastAddress: "192.168.2.255" ntpServers: - - '132.163.96.2' - - '132.163.96.3' + - "132.163.96.2" + - "132.163.96.3" leaseTime: 86400 domainSearch: - - 'example.com' + - "example.com" netboot: allowPxe: true - ipxeScriptUrl: 'https://boot.netboot.xyz' + ipxeScriptUrl: "https://boot.netboot.xyz" 52:54:00:aa:88:2a: - ipAddress: '192.168.2.15' - subnetMask: '255.255.255.0' - defaultGateway: '192.168.2.1' + ipAddress: "192.168.2.15" + subnetMask: "255.255.255.0" + defaultGateway: "192.168.2.1" nameServers: - - '8.8.8.8' - - '1.1.1.1' - hostname: 'sandbox' - domainName: 'example.com' - broadcastAddress: '192.168.2.255' + - "8.8.8.8" + - "1.1.1.1" + hostname: "sandbox" + domainName: "example.com" + broadcastAddress: "192.168.2.255" ntpServers: - - '132.163.96.2' - - '132.163.96.3' + - "132.163.96.2" + - "132.163.96.3" leaseTime: 86400 domainSearch: - - 'example.com' + - "example.com" netboot: allowPxe: true - ipxeScriptUrl: 'https://boot.netboot.xyz' + ipxeScriptUrl: "https://boot.netboot.xyz" 86:96:b0:6e:ca:36: - ipAddress: '192.168.2.158' - subnetMask: '255.255.255.0' - defaultGateway: '192.168.2.1' + ipAddress: "192.168.2.158" + subnetMask: "255.255.255.0" + defaultGateway: "192.168.2.1" nameServers: - - '8.8.8.8' - - '1.1.1.1' - hostname: 'pxe-proxmox' - domainName: 'example.com' - broadcastAddress: '192.168.2.255' + - "8.8.8.8" + - "1.1.1.1" + hostname: "pxe-proxmox" + domainName: "example.com" + broadcastAddress: "192.168.2.255" ntpServers: - - '132.163.96.2' - - '132.163.96.3' + - "132.163.96.2" + - "132.163.96.3" leaseTime: 86400 domainSearch: - - 'example.com' + - "example.com" netboot: allowPxe: true - ipxeScriptUrl: 'http://boot.netboot.xyz' + ipxeScriptUrl: "http://boot.netboot.xyz" b4:96:91:6f:33:d0: - ipAddress: '192.168.56.15' - subnetMask: '255.255.255.0' - defaultGateway: '192.168.56.4' + ipAddress: "192.168.56.15" + subnetMask: "255.255.255.0" + defaultGateway: "192.168.56.4" nameServers: - - '8.8.8.8' - - '1.1.1.1' - hostname: 'dhcp-testing' - domainName: 'example.com' - broadcastAddress: '192.168.56.255' + - "8.8.8.8" + - "1.1.1.1" + hostname: "dhcp-testing" + domainName: "example.com" + broadcastAddress: "192.168.56.255" ntpServers: - - '132.163.96.2' - - '132.163.96.3' + - "132.163.96.2" + - "132.163.96.3" leaseTime: 86400 domainSearch: - - 'example.com' + - "example.com" netboot: allowPxe: true - ipxeScriptUrl: 'https://boot.netboot.xyz' + ipxeScriptUrl: "https://boot.netboot.xyz" 08:00:27:29:4E:68: # bad data - ipAddress: '3' - subnetMask: '255.255.255.0' \ No newline at end of file + ipAddress: "3" + subnetMask: "255.255.255.0" diff --git a/dhcp/README.md b/dhcp/README.md index a28e58b8..686569db 100644 --- a/dhcp/README.md +++ b/dhcp/README.md @@ -6,11 +6,11 @@ DHCP library with multiple backends. All IP addresses are served as DHCP reserva - [Tink Kubernetes CRDs](https://github.com/tinkerbell/tink/blob/main/config/crd/bases/tinkerbell.org_hardware.yaml) - This backend is also the main use case. - It pulls hardware data from Kubernetes CRDs for use in serving DHCP clients. + It pulls hardware data from Kubernetes CRDs for use in serving DHCP clients. - [File based](./docs/Backend-File.md) - This backend is for mainly for testing and development. - It reads a file for hardware data to use in serving DHCP clients. - See [example.yaml](../backend/file/testdata/example.yaml) for the data model. + It reads a file for hardware data to use in serving DHCP clients. + See [example.yaml](../backend/file/testdata/example.yaml) for the data model. ## Definitions diff --git a/dhcp/docs/Backend-File.md b/dhcp/docs/Backend-File.md index a4f9b8ac..8ec86cda 100644 --- a/dhcp/docs/Backend-File.md +++ b/dhcp/docs/Backend-File.md @@ -22,41 +22,41 @@ See this [example.yaml](../backend/file/testdata/example.yaml) for a full workin ```yaml --- 08:00:27:29:4E:67: - ipAddress: '192.168.2.153' - subnetMask: '255.255.255.0' - defaultGateway: '192.168.2.1' + ipAddress: "192.168.2.153" + subnetMask: "255.255.255.0" + defaultGateway: "192.168.2.1" nameServers: - - '8.8.8.8' - - '1.1.1.1' - hostname: 'pxe-virtualbox' - domainName: 'example.com' - broadcastAddress: '192.168.2.255' + - "8.8.8.8" + - "1.1.1.1" + hostname: "pxe-virtualbox" + domainName: "example.com" + broadcastAddress: "192.168.2.255" ntpServers: - - '132.163.96.2' - - '132.163.96.3' + - "132.163.96.2" + - "132.163.96.3" leaseTime: 86400 domainSearch: - - 'example.com' + - "example.com" netboot: allowPxe: true - ipxeScriptUrl: 'https://boot.netboot.xyz' + ipxeScriptUrl: "https://boot.netboot.xyz" 52:54:00:aa:88:2a: - ipAddress: '192.168.2.15' - subnetMask: '255.255.255.0' - defaultGateway: '192.168.2.1' + ipAddress: "192.168.2.15" + subnetMask: "255.255.255.0" + defaultGateway: "192.168.2.1" nameServers: - - '8.8.8.8' - - '1.1.1.1' - hostname: 'sandbox' - domainName: 'example.com' - broadcastAddress: '192.168.2.255' + - "8.8.8.8" + - "1.1.1.1" + hostname: "sandbox" + domainName: "example.com" + broadcastAddress: "192.168.2.255" ntpServers: - - '132.163.96.2' - - '132.163.96.3' + - "132.163.96.2" + - "132.163.96.3" leaseTime: 86400 domainSearch: - - 'example.com' + - "example.com" netboot: allowPxe: true - ipxeScriptUrl: 'https://boot.netboot.xyz' + ipxeScriptUrl: "https://boot.netboot.xyz" ``` From 43d17c58b17eacff588e82adf0f5ad26c722c873 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Mon, 15 Jan 2024 09:36:24 -0700 Subject: [PATCH 15/17] Move mode doc to top level README: Signed-off-by: Jacob Weinstock --- README.md | 24 ++++++++++++++++++++++++ docs/Modes.md | 15 --------------- 2 files changed, 24 insertions(+), 15 deletions(-) delete mode 100644 docs/Modes.md diff --git a/README.md b/README.md index 051e2899..5ff707e8 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ Smee is the network boot service in the [Tinkerbell stack](https://tinkerbell.or - backend support - Kubernetes - file based + - ProxyDHCP support - TFTP server - serving iPXE binaries - HTTP server @@ -31,6 +32,27 @@ A fixed IP address that is reserved for a specific client. An IP address, that can potentially change, that is assigned to a client by the DHCP server. The IP is typically pulled from a pool or subnet of available IP addresses. +**ProxyDHCP:** +"[A] Proxy DHCP server behaves much like a DHCP server by listening for ordinary DHCP client traffic and responding to certain client requests. However, unlike the DHCP server, the PXE Proxy DHCP server does not administer network addresses, and it only responds to clients that identify themselves as PXE clients. +The responses given by the PXE Proxy DHCP server contain the mechanism by which the client locates the boot servers or the network addresses and descriptions of the supported, compatible boot servers." +-- [IBM](https://www.ibm.com/docs/en/aix/7.1?topic=protocol-preboot-execution-environment-proxy-dhcp-daemon) + +## DHCP Modes + +Smee's DHCP functionality can operate in one of the following modes: + +1. **DHCP Reservation** + Smee will respond to DHCP requests from clients and provide them with IP and next boot info when netbooting. This is the default mode. IP info is all reservation based. There must be a corresponding hardware record for the requesting client's MAC address. + To enable this mode set `-dhcp-mode=reservation`. + +1. **Proxy DHCP** + Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info when netbooting. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. + To enable this mode set `-dhcp-mode=proxy`. + +1. **DHCP disabled** + Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network that will provide both IP and next boot info and you want to use Smee's TFTP and HTTP functionality. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. + To enable this mode set `-dhcp-enabled=false`. + ## Running Smee The DHCP server of Smee serves explicit host reservations only. This means that only hosts that are configured will be served an IP address and network boot details. @@ -82,8 +104,10 @@ FLAGS -dhcp-http-ipxe-script-url [dhcp] HTTP iPXE script URL to use in DHCP packets (default "http://172.17.0.2/auto.ipxe") -dhcp-iface [dhcp] interface to bind to for DHCP requests -dhcp-ip-for-packet [dhcp] IP address to use in DHCP packets (opt 54, etc) (default "172.17.0.2") + -dhcp-mode [dhcp] DHCP mode (reservation, proxy) (default "reservation") -dhcp-syslog-ip [dhcp] Syslog server IP address to use in DHCP packets (opt 7) (default "172.17.0.2") -dhcp-tftp-ip [dhcp] TFTP server IP address to use in DHCP packets (opt 66, etc) (default "172.17.0.2:69") + -disable-discover-trusted-proxies [http] disable discovery of trusted proxies from Kubernetes, only available for the Kubernetes backend (default "false") -extra-kernel-args [http] extra set of kernel args (k=v k=v) that are appended to the kernel cmdline iPXE script -http-addr [http] local IP:Port to listen on for iPXE HTTP script requests (default "172.17.0.2:80") -http-ipxe-binary-enabled [http] enable iPXE HTTP binary server (default "true") diff --git a/docs/Modes.md b/docs/Modes.md deleted file mode 100644 index 7b6c51a9..00000000 --- a/docs/Modes.md +++ /dev/null @@ -1,15 +0,0 @@ -# DHCP Modes - -Smee's DHCP functionality can operate in one of the following modes: - -1. **DHCP Reservation** - Smee will respond to DHCP requests from clients and provide them with IP and next boot info when netbooting. This is the default mode. IP info is all reservation based. There must be a corresponding hardware record for the requesting client's MAC address. - To enable this mode set `-dhcp-mode=reservation`. - -1. **Proxy DHCP** - Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info when netbooting. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. - To enable this mode set `-dhcp-mode=proxy`. - -1. **DHCP disabled** - Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network that will provide both IP and next boot info and you want to use Smee's TFTP and HTTP functionality. In this mode you most likely want to set `-dhcp-http-ipxe-script-prepend-mac=false`. This will cause Smee to provide the `auto.ipxe` script without a MAC address in the URL and Smee will use the source IP address of the client request to lookup the corresponding hardware. There must be a corresponding hardware record for the requesting client's IP address. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. - To enable this mode set `-dhcp-enabled=false`. From 3f27fa0e00e21b521cf3f7158f85eddff9dbe3b0 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Mon, 15 Jan 2024 11:00:30 -0700 Subject: [PATCH 16/17] Add doc around using an existing DHCP service: Updated the top level readme. Signed-off-by: Jacob Weinstock --- README.md | 18 ++++---- docs/DHCP.md | 96 +++++++++++++++++++++++++++++++++++++++ docs/images/BYO_DHCP.png | Bin 0 -> 106477 bytes docs/images/BYO_DHCP.uml | 36 +++++++++++++++ 4 files changed, 140 insertions(+), 10 deletions(-) create mode 100644 docs/DHCP.md create mode 100644 docs/images/BYO_DHCP.png create mode 100644 docs/images/BYO_DHCP.uml diff --git a/README.md b/README.md index 5ff707e8..61e0d1ca 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,9 @@ The IP is typically pulled from a pool or subnet of available IP addresses. The responses given by the PXE Proxy DHCP server contain the mechanism by which the client locates the boot servers or the network addresses and descriptions of the supported, compatible boot servers." -- [IBM](https://www.ibm.com/docs/en/aix/7.1?topic=protocol-preboot-execution-environment-proxy-dhcp-daemon) -## DHCP Modes +## Running Smee + +### DHCP Modes Smee's DHCP functionality can operate in one of the following modes: @@ -46,20 +48,16 @@ Smee's DHCP functionality can operate in one of the following modes: To enable this mode set `-dhcp-mode=reservation`. 1. **Proxy DHCP** - Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info when netbooting. In this mode you will need an existing DHCP server that does not serve network boot information. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. In this mode you will still need layer 2 access to machines or need a DHCP relay agent in your environment that will forward the DHCP requests to Smee. + Smee will respond to PXE enabled DHCP requests from clients and provide them with next boot info when netbooting. In this mode an existing DHCP server that does not serve network boot information is required. Smee will respond to PXE enabled DHCP requests and provide the client with the next boot info. There must be a corresponding hardware record for the requesting client's MAC address. The `auto.ipxe` script will be served with the MAC address in the URL and the MAC address will be used to lookup the corresponding hardware record. Layer 2 access to machines or a DHCP relay agent that will forward the DHCP requests to Smee is required. To enable this mode set `-dhcp-mode=proxy`. 1. **DHCP disabled** - Smee will not respond to DHCP requests from clients. This is useful if you have another DHCP server on your network that will provide both IP and next boot info and you want to use Smee's TFTP and HTTP functionality. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. + Smee will not respond to DHCP requests from clients. This is useful when the network has an existing DHCP server that will provide both IP and next boot info and Smee's TFTP and HTTP functionality will be used. The IP address in the hardware record must be the same as the IP address of the client requesting the `auto.ipxe` script. See this [doc](docs/DHCP.md) for more details. To enable this mode set `-dhcp-enabled=false`. -## Running Smee - -The DHCP server of Smee serves explicit host reservations only. This means that only hosts that are configured will be served an IP address and network boot details. - -## Interoperability with other DHCP servers +### Interoperability with other DHCP servers -It is not recommended, but it is possible for Smee to be run in networks with another DHCP server(s). To get the intended behavior from Smee one of the following must be true. +It is not recommended, but it is possible for Smee to be run in `reservation` mode in networks with another DHCP server(s). To get the intended behavior from Smee one of the following must be true. 1. All DHCP servers are configured to serve the same IPAM info as Smee and Smee is the only DHCP server to provide network boot info. @@ -139,7 +137,7 @@ docker compose up --build # build images and start the network & services docker compose down # stop the network & containers ``` -Alternatively you can manually run Smee by itself. It requires a few +Alternatively Smee can be run by itself. It requires a few flags or environment variables for configuration. `test/hardware.yaml` should be safe enough for most developers to diff --git a/docs/DHCP.md b/docs/DHCP.md new file mode 100644 index 00000000..adffdedc --- /dev/null +++ b/docs/DHCP.md @@ -0,0 +1,96 @@ +# Use an existing DHCP service + +There can be numerous reasons why you may want to use an existing DHCP service instead of Smee: Security, compliance, access issues, existing layer 2 constraints, existing automation, and so on. + +In environments where there is an existing DHCP service, this DHCP service can be configured to interoperate with Smee. This document will cover how to make your existing DHCP service interoperate with Smee. In this scenario Smee will have no layer 2 DHCP responsibilities. + +> Note: Currently, Smee is responsible for more than just DHCP. So generally speaking, Smee can't be entirely avoided in the provisioning process. + +## Additional Services in Smee + +- HTTP and TFTP servers for iPXE binaries +- HTTP server for iPXE script +- Syslog server (receiver) + +## Process + +As a prerequisite, your existing DHCP must serve [host/address/static reservations](https://kb.isc.org/docs/what-are-host-reservations-how-to-use-them) for all machines. The IP address you select will need to be used in a corresponding Hardware object. + +Configure your existing DHCP service to provide the location of the iPXE binary and script. This is a two-step interaction between machines and the DHCP service and enables the network boot process to start. + +- __Step 1__: The machine broadcasts a request to network boot. Your existing DHCP service then provides the machine with all IPAM info as well as the location of the Tinkerbell iPXE binary (`ipxe.efi`). The machine configures its network interface with the IPAM info then downloads the Tinkerbell iPXE binary from the location provided by the DHCP service and runs it. + +- __Step 2__: Now with the Tinkerbell iPXE binary loaded and running, iPXE again broadcasts a request to network boot. The DHCP service again provides all IPAM info as well as the location of the Tinkerbell iPXE script (`auto.ipxe`). iPXE configures its network interface using the IPAM info and then downloads the Tinkerbell iPXE script from the location provided by the DHCP service and runs it. + +>Note The `auto.ipxe` is an [iPXE script](https://ipxe.org/scripting) that tells iPXE from where to download the [HookOS](https://github.com/tinkerbell/hook) kernel and initrd so that they can be loaded into memory. + +The following diagram illustrates the process described above. Note that the diagram only describes the network booting parts of the DHCP interaction, not the exchange of IPAM info. + +![process](images/BYO_DHCP.png) + +## Configuration + +Below you will find code snippets showing how to add the two-step process from above to an existing DHCP service. Each config checks if DHCP option 77 ([user class option](https://www.rfc-editor.org/rfc/rfc3004.html)) equals "`Tinkerbell`". If it does match, then the Tinkerbell iPXE script (`auto.ipxe`) will be served. If option 77 does not match, then the iPXE binary (`ipxe.efi`) will be served. + +### DHCP option: `next server` + +Most DHCP services all customization of a `next server` option. This option generally corresponds to either DHCP option 66 or the DHCP header `sname`, [reference.](https://www.rfc-editor.org/rfc/rfc2132.html#section-9.4) This option is used to tell a machine where to download the initial bootloader, [reference.](https://networkboot.org/fundamentals/) + +### Code snippets + +The following code snippets are generic examples of the config needed to enable the two-step process to an existing DHCP service. It does not cover the IPAM info that is also required. + +[dnsmasq](https://linux.die.net/man/8/dnsmasq) + +`dnsmasq.conf` + +```text +dhcp-match=tinkerbell, option:user-class, Tinkerbell +dhcp-boot=tag:!tinkerbell,ipxe.efi,none,192.168.2.112 +dhcp-boot=tag:tinkerbell,http://192.168.2.112/auto.ipxe +``` + +[Kea DHCP](https://www.isc.org/kea/) + +`kea.json` + +```json +{ + "Dhcp4": { + "client-classes": [ + { + "name": "tinkerbell", + "test": "substring(option[77].hex,0,10) == 'Tinkerbell'", + "boot-file-name": "http://192.168.2.112/auto.ipxe" + }, + { + "name": "default", + "test": "not(substring(option[77].hex,0,10) == 'Tinkerbell')", + "boot-file-name": "ipxe.efi" + } + ], + "subnet4": [ + { + "next-server": "192.168.2.112" + } + ] + } +} +``` + +[ISC DHCP](https://ipxe.org/howto/dhcpd) + +`dhcpd.conf` + +```text + if exists user-class and option user-class = "Tinkerbell" { + filename "http://192.168.2.112/auto.ipxe"; + } else { + filename "ipxe.efi"; + } + next-server "192.168.1.112"; +``` + +[Microsoft DHCP server](https://learn.microsoft.com/en-us/windows-server/networking/technologies/dhcp/dhcp-top) + +Please follow the ipxe.org [guide](https://ipxe.org/howto/msdhcp) on how to configure Microsoft DHCP server. diff --git a/docs/images/BYO_DHCP.png b/docs/images/BYO_DHCP.png new file mode 100644 index 0000000000000000000000000000000000000000..68f5148db1cdce592579bff3ad98f168885dc990 GIT binary patch literal 106477 zcmeFZXH-+$*FLHcAoM_x-n&W((tGbHD5xO4qky6)gbpEe5CoJ+RYb5Lz4y?2upzxS zrT30^#dFTDym#Dj@2CHl`;M^{M_lZzz1Ey_&G|gf+)+l?v?%~*0b=QpVA8!{*U7&%T|L-fL9GcIA&AG`%z|Hwz z*CweB;;v(4F!aAJ6#@cUgFDRsV=Ol(Hb?Tme;tf?Pm>UFQPQgB4n#xcKdx7iP#wgD z&knBd{>NRwcpJUAlmhzyo}wT(_)1NQVdfA2V=DI`yqFuaZ$E0PK-6Fdi&;Uj%A0y` z^`7Ueg^_XHF*{Cte);xD(C<^{n;%SypA4D_e3CMGkAUXiDepcz-JdKcZmF>e;Jq9u z%x4}y7V7t@cCF@C^`m)o-`UPr!#yks#}I?Hzipa(#mm)_9+8UWg4z+^R0tc&e5XAQYtG@c#xDAw98yn}TVibVxO=LF7^=ANWW@7{rQONQL~bZ2<3D+#_= zY}*n(n4=3{92456UvUCx(@NpKO;Gq{({bW;|?Xs<6SL z$^&n|(V|@bQJ9tSsQcnI4`v6N5SG8Z@@yo zae^HpOALk>t6uv^%tuho=W#F{F&ZP&_r$E@c%?M{ z6g<=3h-=~VlaqAM{$eim@El&uNn2UefR3a6>$+=)$Loy`D48V^Y%UeTa{|F?^6~=X zY6Iq(@Q}a?Vd*vO==HUnTQ?})^y?L-5WicW(om%nitAP^!2OjdLJ74t!*V}%FQY$Sf7&R0%D4*^a z`)zdzWOzq^d8^M*`}^#8J8&)VaGt?$;iYrB^Tkf?b&}lkz=h}(fKi~b>?n;O-I*%f zSh^L`c782^Ol}3OC^F|4Hz>z#_m#~$jfag7gS;IKg3b=}ZhpKa>^i=&Xn$jR3P*JF z(;Z3I)94=_a@+GnUw404udB(=K7XwqF;cylxg03zF{QFRQtsO2cpU;y_nHjw7C74< z^EC9^?vr*u1;am*13Ue=TGT?dH~OGaRxB)GyA9W4AFt@dChrwH`a8)`X_&M?V>Oh!@FNHDR_)4*!;9em4`G?9!x>i4B27;8FQj& z{Z8We!wxnr#_yy8=~G@yq^m3vE(!86%>_FgK_|B8>pr37o{RkzeNv-UY)A^{f}q6= z?{qG|A=?PEETw?KE3BJp1MNMcjVsN0Pi1DH-d5;RZffO|qK_$^k&gVE7E{-!js!B1 zcAvU3t~f@I`z&UxE#j1p-MY^2N5KVZ#2R-8&HDv;4JaNkExIsuCOhzJ?$tQ)2Swo-$~ZbrgyPx3tk=lh zyyv~^pS3wMiaxf4Q~0sWYTZ`%<);^ej5qR& z!|&IwN(t21k>gl!^wXixgHS5D&N~%uKNmQWg&H`%ABUyp$RJ%W8X*O9Ho3PEiAK^hl9C4U);^u*;VS{_1p`UTywI?{*<>xro%$WN9iP~?w1=i6Z z^sC5$82jaQt{e!}Z)tV-F+)Be_?>1HKbi1Z99E}RoE!1m`K+BWjk7&H++LI?OhPd> z3&tz9&?&%+*Bg%IW-nH-diIJ***2pSIf8(v^b58kp9(W#PEJtdA|BxZ*{FegpR>b` zcm3AuD_^L3f6*0?EA1Uis_*O)7`bm_*J}Izve`32g0^ z>@5ajLXiPeAtZY3NZ_M2MiT9hmOoX>W6V6DL=ZqV)Xtp#SIF>AoiG=^sgf}p_CT#N@HZ7r;|)P z6a#qCmyuF%5UtxlZBI?l>*po&V_+q` z>eEVe5bG<3Txto<(Q!l?^mZMdj`buscr!Mhi6%ZWOq%7q)#FVCkA~@W(=8aW+jD$hc_^4;0g=lH^&~URl2N>7{ zuUfyIrR!fv)M8JnX}yX_H`Spw$!*N1Cmbz-;9P&)sLzk7A}%CF5x#CIs1#-N8jC4$ z8@|H8@7>82A@k-+X2&AP6-2sdbxh%01|Y_VUw-Y`)i3f{dR=vcy>5z@%GYTln-kG_ zPJ75|fE*&e-u0@Xk-jk45H+QELlukWY+_j;_n*?h9;1gYeFCPhm;@&HN!jZKoJ;dV*>Jz#QQ?l4gPTyekTOBeMtE8sh1p?TpS{uyKXub{+n&r z(8$m8#w8v6j?=yco3@Y3Vh%)Ct2KmLi_`C7Z)u&iKBr=Y&O{KNbA9?9L0Od`S>5yn{j82X! z#CFHIKpe#GE&c1Y47K7O~8 z^2)bpbh0)#W&Oksv^Q;sGZ7!G443}+b^9)RE;3?<1_C9V2vK)oB31~mV-PQvp7Kz5 zt!N7|9&pjt@p!@Gl5W2=y=p7qpO@kFs8VlE7Q2k~p?(0>qA_4BV(jZ83zvx!wo`P# z`cC-GCEf2$mF!_!S8}5weAzHN+`^;blA_!(SIS9@t3YDWG*89Yg;c#>(it!=m*4yT zsIs|52g=UZ&BV_UzNFrkSiEpT6;`&8F8k)7Ja%6|iZ@}!D_yrGlvYaJVL*KDxMhqx zleBa@`?6p%{1ZH=qL;ka0ryt z$)64v45T8C>cN6MgoD&_?eNEvCg)`c2zmGQ?%1g{BOdwR2%egLRN~n7;tM3n@C{o& z{4xf{i%zv|vh8f64}vM>+ibC47L}7BL5D<(b@(B5VpX6l?4)FiX)|xL_D;Ez$%uiQ z#~N{T)2Ho(_!V8o$-Z+4TX|CV_=0+B*-$axQu)Sfx3TyQsfE~wpNUaW921IEl%@UB z1W%CVz&7x}ItTh;S_59^4P9PT0IBOczrs5MPyuxoZm}gr-B$UfD0C@p^IC!VqS zW0_iKhO%>~c*;8uC9zuxdEGbu*osO5-daNcSPPoQVC-8K(dNsoL?)-}MDS^3BrAnD zMcI(r5cNbs5$fWSB6sz@(cn->*R_4NV~8W-x&-^p{4BtL&jPlqK7 zqPk*gHJFC{tJwCr?glGc&()LWFfEQT&Z?`>>@Bo3%ZP_ZM0r%odM+eWNHU`t5*oM39mV@5(fy83i6fb;bRo7Hzj)Ki|xh?Qh^}7c2{NX58D%jL8WdAi`Ij zunlg=cGg~ciWA!NUyHP93|7ARwJ<%nzH;p9-^spD z;p>Eml+A0SQocJKs@59uVY<3WW}jdnzEX$??k4jUrIHJQD{cH1M3FvVoRDbOIS@pB zan_>f7j6`YVc`zbAVXIjChB4}^n3>|%^Lk!b%&PCKG& zU77m74(XQitxLXfeKEb9Gqlf_X|px)9clYm$+6z>2u3l>*DJQyhU=|VZduII4A8=d z2^D@&i#wvSM*u8f8mfZG@IdUuAw(wxT54m4LiVCKPKGlb!p%&uXtAZQlu0FwhE&da z2p6%P4SJ&+GzO~i)K6wJNVhVD(af*Im1dcot7Ql^a>be>J88osdEBpvTD-MEO5OX` zc`CnWDGyF9u4jzLk$nBTQ)WDGIKteV83!F$XVa4mS^C$Zk`zu?$a8PTUsYzKP z3IcyQaZ(#xMG`Kt9h*nRrOrk1bdK`u1g0Frjd_PtNW7GRD$=Cj$~Jvd#P?ibnAU8T z@dVxTEP5ivfQ&y;`XGk~x#NF@cI|>f%RDZeC`{PGzXplsv|=4pGs+k66R9@zAVmKb zcWS+ZfJkQAkcDp(!C!1BNgQy?`AD-kpP<$`XWIr8S{fkYHZzKC=K0+CJnSVvJCZLZ z-qVI?yh5uCmCZ2TL_au8$o;cU8YE(6ofVp@IU6MLMnyT?7}VX5cS9*08v-Vs2K?GE zvg&FF%*~xyT~H?v^(I@3WAzm%vxdq_HE@`y1|pm~He{ddnL?;HxsN&rmOkyvNEYRr zIGZZ=Ork7q`HAD?5-v=lqF1CE+Q}_->m4fU*q3YNxC!Hfy%b)bB_F7zMm$8U&rbwr zp+VFqU5+m z5ArQ5=7KoNj^9|jv9JJt1kRmvl9h7@_yugjsS>>ZWt$x+Zf zkIgX2mQs2@#WCnJ$JN8cmFk@zhllgOi*4VZ2p!ZNmG|I$){yM$02z_2c{(&tacd*} z0#0_Hi^txLi=5wT7GcKyrs$R>$bDvbII$-Fo&_|(MNc@+H&;TlwqCm^)q5i@JYqD3 zDwh=V>5KQ(Z#2Wy!k$HbHauF=c-f7hs%9_Nw1aAB3`u;oJz6a^tA6yh$4%SxLZ+zK zDox4ATWalRKAJ_GJ5aM$zm_Z{6HQAwX5wZ(gWB(nIZg-CavlwdX9qcxPysZ>@Ak`K zg1GN>JgJNQ8Oujc^AgW^f045YSm^&IrFlD<_MpMXcgjQ`pRpyv5^TQE(wwIeWF}bg zou!;ghrQOrX1FvjK=YcvCJJqv!5FpJr>z)0&WH*%2~7wu`~~lYaC3sEge2v2fkN{n zUS*?>MwcC5il`9|KNsMQ+~8F!)bUDKvDAtbwW94A=Ap`gH@BO`SgiKDJ_V2;KI>M2 zl&GtqF3MYc2iTyBEFZ#UfO9rqy$)rXsuXaKHexex2fG5`4B*NcBZ zM3rcevEP8?_JDhBWu1FDsRO~j%y8^^QN1(5DkR{SVOX5*mW+I%6Kb$^E|EVW++6eT z@6>p;fk!YW((h^D&yxSRnDC*2E8(8jKO_H|=KqfWdzJqGw>}Bt*d2f)n&w^0xFSIz znM8%hgUvdgvVDF30S~Hxi%4hch57wrpY4SUQ=Q#`I@Agq!8Z%7>Q^4*%P+(>U6mep z*j~iphEKOo+X1A3tY%ZYMq~`q=Wbz)-O><1@c6MN6!1@Ce*haD0%$I~qA$AcPResF zo5QVnfSp%pzzn3vJk+1X$%(hPe|z@~)Z^wa?+THB0*C+kdkqp7vB{$1Zv+3_U>Nvw zH>)#gZf|jK$_s%0WL+f9g{(lkw%K#r<5)%k=##AE_?_aa&DLDk;3sow?|O)_i(8Em z-sYoGDn^JnCh_)P-`*S3&GvuetbP2`4xq*!eYyz8=zC{na5@9~j;H10-jj+!a#?RO z2Mety$IVa|b8LKPR@(6U!w#vO zb0^LH=46Dr2oWXCg(Z}DL_v({UwzQU6Dfi)aQJ=Zzk#~Q+~{8@j7=V)&_jex@nPu* zZqSQ7v*{)^uFijn^Dh<>{a-;7#_!ve91xTz5{34bv&br~p3mZLm zVm^%8MxqVb`HSQ!_>KLLN{4@Dfh_OWAp_Mu6 zVmVW?4H;G+Ja>+e1vfUC=-s^=_)R85gXFm^D%mEwU3Ah_AMsWy6f7W87ZDbm9H!6v zzl->PU&PDisJqQ*QjiG?j^547SQ#{bVEIwtTl% z3xMgv6MpNKTiYqtfy3bxOm^NAK6&o7s}-&)fdGAVAFFyOh1{?i1;F0e`mI8%)q)@Z zg06h3oOq<1R}Uzzf}HUSmv1pne#e%q)hyoup!FvJLRNggW$x&9g%5Kj{E8?`cdm^u zKqptgHR>_sa2G+~|h0BiB)s9G;z@>Ctlbxqyk@w0Uty0(-P+$FOs9c_x6*vNzHR&`3pAxpfgXujmBm5kl z%3Xhmh>4^Kn=e4Ci#I4M6T}i!^$g2ue1& zZ$3|#wgDGt{zZf!N-zn;b6msep7}8#dNORH>(?SU0zIq>@C4z6|8`$&?@NfW7GlAj z2LKF1spb#r%lk4FYtfX;m4GiBEU>JV%>3w&62Q{gRg?&0o+|E{E}mtH*=y-OI4G(E z8)PXn5zcbobjCInP%m5s9zRP7H6vBx*%kHbjNf@TlPVod5#CN|^V$7=TYj^ZTWKbe z$8Em*`Tg55)Jx=|bpr*BXTIK(a6^U9)u(_dx`F3Fr*}u5{IGN%c%|BIr(L!AdcB3b z;bcp{cBQ1p4ICd8ZS>|tcq-`^p80}&p93+C&pGEZFbIuTrliSlZBPJ(k8uYUQzqUo z)3S=mRNro1XZ*2_LZmiTkn{5mk zlrfdRf(fZS`d|g!o5_Io9Lfqh`|0)t(8nKnS?RUU?TH0N_TL*Wc{}dWLb+DmrSZ&$ zOjT?Y^%Kw)rC+bf_XRNW+!Ji|uW1Vk1qi+IC9HH+5e9Rq<#$1Bn`GNSitesdlnCofQQ_I%)Jh^*S&FE$2mJUp6#{DDpS4)3!$LW^U!;0nGI=*uF#B7HUmkz}nV)82 zvYq-;+ih@djsm!vJvMU z*T$;*$-};eNJX}9Q9%OZo3Q4i4FP^BZhZIfR)-ixMZsk0snkL>Ll~z+pJ(vbsahdE z-%OUMy>$|2I?d?%b(R z+TO`)Mf3Gi7tkz+2@3smT#=}^ z4<;M#N3a%~yuZQc+-Bir%}}yIa|4&_mNwY3p>q(t(zve{Bx@x7_0;qZ!0hH8#_6n|-w6 z`Wr2g>-vZ!zD3wD63XtQ>s`3 zu7P2_$Bd)8t1oMfwrQU4>Nn0#s6v`;>A?-kx;U(je&RW-o=0kpNNA1sdIP;Gebv5P zSX0^tEorXcj>+vm4wQy>Adcs%L)81%<|*JW_xHxVt-WtutboZG4xIpUb5TkJo?+X) zbUbJ{Y}eru!J1ULRw%>? zv7kuJ=?o3s7M$Bx;~y1mUgN!S2^F=HoPKrd=lS}Z-mgJzI|W)BzxAO)o7ZUn=020*5g zAs@R&DQ?A$^A?(X?@;s>Aag^VbCy?E3TCf!-$4g?8fC0Mw%Y09I@HkRC=p8nHA5(< zR$fwK(eF5ANnaz=-*<$+^=J)Ekn3;uRGN&K3)@9J^NH86S5S9IntMkUJ8Vl6tS=YTNgpBhY{gWJREXGOx7D`fU8jjQDH*?=;`J{= z8f3+(Ia^?pc4LEQ&Z}r#hgmsi1yqcK*mkW6OciE|C%&5#6h`)S*%j2#EL^DKwEJWl+?lMj(STlT>vGeVsTK7% zv>*Dc)E05t=G4{GjmZY)ZWr&c^{P7uoTJMnIy))#0$trJL+!d_v7v2-(o4@J4!xXm_BMmxK7 zP*YnkH=&>Hr#%MZ40wixsx)++ubl{rjtH+h*1X?SM_s&WS7U|uqUZ<}Q1Midf~@J= z+w1(XP{UjNjnc+HYazy$%E;D7+(`Kiqpt?G!tEk%&pTu%=n$kOeLG8kzF01gZLY;M z9L$jK-#2!%X*I(K6KLZ6RYcg*BUMnmn83ciQw3F1q{=Pm+((;kvRDb`=QetTch2AJ zs<64lX>wl>Y?717{}wy{TQUAe*?l|ThFQrU8F{CM_fiNcoZl2wHx}aYihCG*jdm~T z!yjb^zH|n71itX%wC{adyRfIC+so?uYG&;xo~SN z3g}d7*CYWoG66(|k4C>g0o7Q&978=hS+2#H^?tI=7iK(U0eG1tfT=`XmB0fH2lMu; zuWY(I&K65>v<9S0`iS{QW@qaYTZs8Us&9mULY^TgFn7e<>=gi+HPSOl$BK znk$Y|5DzGkEjR@898ToZUs-w+-~Z6?OJe{KoX{nvC@FpagBPH9f^B?3h1IiL)gI6B z002!TAQgDN97x3`XTs$(?;Cz)QWo-ctlCq2(u4Sp|Ngl5DgcCw0T$%^s(eLa6X@t1 z@}?=uQcDx+D0UR{u39|m;*+{RCL@q)g3sRnT;1VKprUXA3d>@Am~-@0AJ>GC1vhCj zSUTVowqu2@>Z;CZ8SkkB(MwV_#5i@z8S^qoW|JyoP zHwK>Kt1GF8Gp!QlMv{1o!CxUJq!JG%#;(czP>w~}kL39-$XMH~d0G{UtRIr1&SJVE-%wt)k`|tfIa0dl|Vflvw zhLCK&;PE4n6O4*=uo{<9WK&q6E?)KZrT%X#8t}t#HyMA==6kgd5I0}aT!};AEu`1J z|2c4!e%9{FNDpV0H}1el;|ct$N6*VJZ~c5@V7B{D-Od8 z%iiDmzRPV?263p=`m=`zpa>f?MFN>(0CV}xe{nXPWFgFw_wZ1aGcxT@7~&iSfl>{Y zDRDQEl0#{1p+CdbI$-xX7v|qV>XH6AY#D$*%tMJO^MQDEu^ z1^%R)#J`qDNK8=#&?_?l8+8Y9p!b%3>>&?_9qHbesMo=;(P*-+gj05?@RGFG&kM<6~!K@2lK-4`~qIrh?mzs z`YQTF1JaF`wo8m?iekWbQ_IgKsITBpEI_#o6SKI%;@;i+qnX05XEF;xy6XtYzTH68 zP!4_$UIMl8{iEEckt$|Dff=n;0~CwzTkz1wK9?^LziPjEKk~1THUG2;Fdul8b*cPX zRf|l|Yi@!zEj;ue%TRA42WrkSzvo)9j5KjM|M8vX1bLahAD#vJ@2w2aSH%CqKpkT6 zGcM5-#yIt2rmxH>0xkC-=n-(?y=nf>ZoL|#U##^T}2kjJ#Id?lcJ$?a6E%hYWWQn)r)k%RRdk^o9ib3 zv6zNaKn_|8G}p}lddzq2ok163@MrBxT!@l__HUSVT-g?Z(q46|o`2!m%ld#P&k*cK zwIt96;9Wp6un|Zf)5M*h`o*_Cm0xoqVMo3K5VfzabC$`^90LJ)Bez3<)~Vd^3!Pe81pBi*J@*<_-+v|;4dSf$akLYh_7Vx8sdVF_uOFt>e2#9aH`6AtuXGh z-Fxx?8H_7uTc|T!pQvZc z>k+9Hx-T70$>IdURyiQlG*_K+j?PxDOsWF5EauybOED@+_$`q#*Sw5D>|B#yl{f~C z0q!3^RYYrvIk@-halL?D1cBB=SNKc1)sycxUCXurZ6W(iP1WE!wS7+tK7sYB0NE?r z`}ca|?{sYGz3-PmDmlXZqaK)&>rRNZ(#b}P*Pxj*(=Oh{87iiF@bNcZ>FAxG^UXRn zQ~Ns6APqAxp&uW%n#o0nu4}WE58?pg?I;gWdUaaix}F1-*|?p^S(6^H67IcU(WLMd zjQ!JM%^3RX6bXFuOaHlkjZGZov`}ObfYM|M-vPR!z-N%k>>UU+_M><*fH42YTRqqD zBRn7vux>nfzq*k20N;ur^k<_SzVNotPb+MarIFzY2sqo@=Z_b7y{_^u>wkd3bb9CU zjXGZ)0qc?Jw{Qg3tMbN6j{l&I*E`kQMh-BmZtA~x5a=PNxU z0VHi*jVhLIz4SvJl>eyzr0u>%ct((=0KuUiJ|6e#BA$oXs4?0=;No$Mf(NPr;J_RK zIR6U89H5e_h8i0%^R!54Hf?6o0lY93>YFB0yq&F#C)n6>QVuLH`>GRZ;J!I|~Vb4$ZIOAl4(t@k(DHk`Piq@r)w zt(YJj+<=-Z5LctsrBKV*?w2pjNkjh}U~hEp$5{Wm0ekbq!ZQwDJ{HgBYxOi1mcyw* z8t*Ux`xt3JVYjBgGzMU`RfrIZ-tcSp^UI$AJX@fsa4vEZ1lcC2B?Y|~HXgL!m>1cE zY+ot1+CWa(q8OTe0NEezyah-Jit)*zYSPOXE8m}d(j2{TS7B2|>VPJDy?4~Crqbwa zZow&X?O5i8kPSW@WN^ZqSYJ)aY`qN2?7)M*8hLhl%wn65e6|iMit<+aGy>$4pB%e3 zF)6XiRy#b5z@a@m*q;$tFN8|=)L9~@!@K~tSNl2xITLk%{Gk-8nYFm!g^<~d48F#` zZ`?|r`(=T*^U_4B&6@r)IkD|soaRdU6%Mra7Lz8inDx@-S(({b@v#Lxxw<4peShK& zv!*Bb%#nN^eN~}Q;&euQ2~cg1CqxHjaa7x#o|$fA#aUd4Ia!A>C5afA(?XOIqGMw* z(~bTRck|3}=weos!ZPwOX@vpBGyKJk5obL&OQ}!q;ukT^;gE0`j`nn3k}0aa zRf}wYMm`t^`L&@SMx}r?&NaBmv<>po`f!_>mpK7azYo%1vrjT88GP5KfZe}#z&7pU zcE(jR42ij5@V-9)vmS-`0T8vZ@E?_lD_QU!ZUi|)IAhD~277@fCP1$|sXjtSP zzY~`b6B0{C`~vyS3I$=muB`&MAnY16oLr=;$)~`Myf%B~?CWi^Y_eH~*@wXRV@D+$ zEH=?xGKjsyG*J`LDZ9&?@>#?$MDJ=)hYM*5Yl#WL#M1L#;+dhGYLa$J=evmfgI7&1 z&9&H+17kzQc?t=#*^VJ6Rl}2#6u!mD1T_5{L!T2Wd(HHR4C6-5-~xLP`A91Lpw8`UZk%7P{;@*jDMbheP9}>`DTnSgT5xKc@p_+l z_oDFfmmsOlVRb?2{Wp|HsVfu3y~7rE_i3(nDH+=>b8ieo?2OTYpRJEt7G)+$igYFT z7tqBeQiuKy>Tx)YOyNvvCG}Ix%lKCjQR^tsqG<|bG~?k^lN<{1N@aaP6I*8e;%%vm z`@I)&nnsAXa{KSoq@r8_L!OvBLK`vz^d`yn*g4#m$616ZuUFBgh%I`Xk2P(JwS@&6 zoJU53erS9rN&o|0+?G#^B=t%ZYeHBYqIkYRLPCm}gjO2beWecSsflKLSl-a!E?D;> z#bJb-RD&~IEgsrMQ@Ty>-X=f^LOGw=sF`xQY{+o7+f3UpL{)cihH+T0oRKTIlpPn_ zd)T-<45HwSy+P()8XBXzs@~)h@1fZq^&?s|$V*$y|0l`sJdST6mC{NsxCo187InxQ zIF~w#x;<{X={F4#Ek&`&K~NDdS(my3XPMxN(k!LA2fmY4_n74t zVmj(g(*;{SSwK=`48`1DQubVHBh54b)KcQg^9@Z#B#y;>!>uV^a=l(j^#znaR-8f> z!XiJ1ytKj29d1uPVLmAC&h=slW1}=3xisq?Sbyka&km~yoT!^3BjZ22x+L_(=LI92 z%Y9{LJGS^fZ69g4i(3>!Vn@~NF442(mFh?khl%pV0t2w(;fq}nChY~|4)}0ql@7rr zPJM13+QZ#0DopQFH<)I_)~c%_ioJ$fb)@c#VouUaGIdH?wx^WUkrOiV-Ko%Ml2*7| zsI$=K7UFll_rjYpT(;?gD%J!2j6wqXsm2R;O<9fkqq)?raRcHRc*i2+{}83 za8>~Q4|VnVt{VJdUN-aACu`~*0fMqWI2{hJO)JHI&9c>dvNjtSnRBQPpcd@=R4AP+ z2xhG0{7Q9VBVBaiQ%yGZc?>+tE{q%V{S3^tQH+U5DgPz2yRI^sHr%|iFveK3*gLTn z6g&zSkX;dsR?>Y%L9ZCYx^7cU_!3$X$Veh|#FU zksd8|-Q7(EZ;*d@u=(^yy4N{FW53{oPn3c|P+$H{yDwBz#KaIhlENW_do2>^IF~^shgk<5|gcxTxY*5&>zSy%!-t!4r z>X^kkp(af>U;Dn_;GerPT#8=8V))BGGYlg3+x_U$hT)=QRyUMbm^db`%WRxP9OC25 zbf#54r^b+=>cKJZMb*vVjwGCQL5!k3(*lStp{OXy%q4NY2@9^+B^KN{9Kq{vbb;5F zZ0_36$U%4}IDRFLg~nZOW=*=~c|G9qTa2=gH9XWk0Tbu{8BLKL$$Lx)FXkT-uf349T?7aD79< zXr|9x0gFDzw;4`79qULVs&D0St(h!$C^n!$6y3<@Ct`Q3VZGTqqe@CIMGAG_<+=$r zHCAl6wAG0aN6lqb97dUjWa`B*WGeQL6m@A#KcLukn0=b^_?#%)bcDZF5fjS$i(d+u z=76cfT5;Fasvrz5duWr02;Qi_^HL3JSG4i6FPv(TXVZUsxWhAkV;}XtOl;Gi=B%@3 zu{PvLX-w{=4YvWSSWh^0cDiI}-ep0I)R0Z6WXVV0crJseY|zKIZvVlPW5JWJSCZDi zR`4e?z?)40d?rh8z{rQRJy7G%%xb>RPb|Ss8A}z^j~3lQPTzYgybPKLBh(@4D|I2= zDazA#{X>$(aDJ3^V!v9#wvlT8jk^Sm6Sux5;1Ho0b zaqejQj?!a_X)F`TT19Gm=^)lI8T6N&>QJO7Bf>9i%9NG%ou%{S{0Mf|YR0F9UFB(LLp zUmEtrbM;L7dTnmAfxJ z4h(Gh)nc+_%f7x5V)|{1{d%J`Y|pLvaQm(m{=9!FF%CE;U1Rqsl})@{xY%aPnF~Q6 zMl~$lrn1RSLLuEOG<1Id;c=kPmS-J3GUDLN9J%}+QRpTF-p?y2j^#&H>jFD_rg6WAH3sM z7`oVA8mgl~ao`PYQ$lUVvT2YKQ)r0*P-e6^rb}_^Ck|mtu|X8KVVl14?oofIK)?n& z|JqOL2Rx4uzG4Q5oS=sx!hGsKE`nIE!*lHF&#*#${b zHjTRtDmg0#s@l@yp8E*fq$17UcIIRyCg=pq0twBX)Q3;CMa_ad`7owDfu6xnu*`Pe z>1e*Q6{`=#{&n$B9t=FoIZw@HmkFUu=A118p$@y<)t;e)5OYR0U&L;ksA5}7(Tgw~g~f&)Z2^UNT|(EIWvb1kRq-&&9GlGv zA+^lm71go>p=4}+E~rqP=>itTNwHC!vn%WbRMYLIGkgJ4wJ5hwu`hk*w_|Z83#6YT z2j*&_*L$3)(^!kY#sy8PH=-PC)Rv!FKCcC&}#qle%2w+3#56CIj6 zv{QeUp#fzg3zxGVTrnY;9kpkmAeKa7Ip!AiVT8h_{%#84K}ALp;(C#MWVY-IKxuQ{ zQY~rdbT=%+uFmM})kC)f)^?$Aip>qWR+#;Ub25w#%HmagIHT#n)xE)cfO2V;iM@S5 zyWpm}p=h+B#`w&=o1xLT`!|h&MSLEACw&WJ%k#~xFm}*u>dIJ`Q?f^-9&CBzmxnMh zS-9TDKKet^$7rr#u376Vy~8{*zVRkkuypAUTCdE&KmV$JWEe{wn}8&0i4&)c++k99H4D(k|Xc4ApY1OQ;)?jhd&g<+ph* z$V-h)!e?}%>3w#62ROolX|o7P93$T!V;I@tGH){TIzD|hHQa*K-Fr!!PfpmH zZJ5|nPlHg(s1I6I2LYhEXJj+W(2XvE>A6qv7yM zo2oy46tuRQWu`5 z^|3EP#@3p#3L0G^=s?4>^Hlu^U3!2Z3wofI3=QCM2t_;{%6Hp5(!p^2C}v?<(0t=x zeT4*F%ICYUcUNetJw?2>Z|8HCwMjq!I*y)3Er#w@F=!cN6Hp(8?ThvLf4=tZES&AO+z&A) zuhL=44o`-?6)U;@?#FDg*ZR{BBxbIU#9P9zFXKLru|g+04{;`SnViNuRZE zH;+9EL=2PL>Yg=eLAk?Q#oUAr6Z@--nRu3Iw2)5r%7xhzwPm>EA*JTv^0vkEbr|^t z;lX9r2X5#Nu5Ylkz^63+8wPt!y^2iA+pUZ8?uSM;SyMz*$jk$ZORJ^P4fu+x_hz>1 zu+j9%1vOC&LJ;o$k&;FbhsXB&u6-us|9y?uZbppCge*Ro!NAt~sW}oN!TlGJ%XvQQ zJ7Pk9Mnexm_7YE!4-4PHwo>2I$rWMRg3pi!hd}P{DuY| zPjh1^x0yjGFX|(ivZDXthgEFhGC&VtnX5uO`7TmL><{8R3Rn#t^*(Z19|l^{8RLKE zK_h5Da8`caVV@`I?SGLIvC27x^L_K1R&Gp`pZX>Y+99vP&E|)Ck8gek-FM7gTbBs(O0J&eXdZc9A4A~DT{7)u_U=qAQ2u~D5SH|JQ4^LYg@T8iRsT zXuJVl<^|rN#L)gqf{aGz8JJyklbuM4&5<&vP)-OT$p~JW?120C(|0}~5N$ZfhU@C> zBus9oxx5_I?3z$**tkuJfpO zevkdvX6KUn17KLh{Qxj4ZshbrMSwS9eBb)l3oFI_s9pa(pAx@IAZ76Q z!2h?A(%>}{!FeAv_3=LV4K(BY?|}bv>+oPO-wdeuxm|&4{g1r|9Ivh4N^yjw+YsXW z2Ow{Ze$fQj>eY{h4JBTyBNv221qoAZqSXwa{HvGsFPQ*YD)2Q`!$M+#f8x+z4W_U# z@Hw6Dhs*!{vC#l9LNZ@pNtwU>_#Z)!ApWrUTL%00XCUB*p!fkgx3EJ6UaXL7n00@) zJx>3Bh%26Hv)>g+_dsR%tyTXct~lCXe^A(XRs-C&!i7nLsS4<=c6QkRf^q(`Tn$dV zjOH&>T(^yu+NCwPy9H7Lc=`wM7B~DD6$o%LnyL5qKGWif$N%>M&xnkV_f}UyA5R4! zMKzbB?lDk?T)O$eG>q&*v{N1PyT5L`06jTe%(?~M#`h73%omt%>T?@SL2eaT+y$s` zA)YrTQu=bxVh#r6Jk+K)0HV7hz%uvT`gA0GL)*mcA3psbY1$Rt;JynG`{YC#><)-z zL&PjAE-WUaTzb}IMQd_IGPyeuJL6w~82{F%r^TNtM)*&|O^pBC1V|)e88?eoi?tP9 zhddgy9bbjSsCk$wxA-PDo|0{T^|40Il;ES89UKfn)J+vy>9ZK+)5eT*rM$|?hGqb?%@*^%oDG1HxP{R`!FHJcj-j{+uWo0Kh$ z#Yr_mLl8;efBumEF?yimfS0g9M=eK|@znKq^P49=clj8!&sr3z!eZecR0Gk{WQBaP zy-@qFkSLd3CFl79;rCgLE(dG+p!KMcm|BUiL&b98*OF`a!(|yVjwK9CBy9h2p?(8r~zHR#Xmsqi?F4du<#;cpxFxjY@!5r&BEL@lvATc0n> z!6kmrRJ-&QDbj*linr34)NwhaKVB_VacDfP6K0Nq@@}I}l1o{QN9TN1c~^P7TAj=I zsJiiua>r?Hyz0blo0)9Vcp$gMMpp|xO|_YCS=vr5Tj0~@)u^60nJ~6JG zw&z+F7B9P#{j6+L=2Za=14k zt!94R9lB5=gnXWt`$ode|}`t;o&fWtTo^7pOC_d>5}L$bWS?Tl?Pxt7DopTmk-9tutS`FGEW``MnM)Mf$ zic;wL8zXbEjypLy5H#Z>t~veFVCLC(y8k0Sd8{;hM-o*jvb7GhS@h#jOgB7c%P}de z=b2`zU%F4iI)B@&)JjIP@~0T>`=y?r`h`^=fE)ao^B_p57@=3q(X(H*G$Yjcl{J`O zl5b!2on`Tby5pq%+F{V6K+a*||$Dqwx={v{LI}*&hrwx6kC?cQVe^^{k0-#eFuC zd$1*odbGXM!?5-<+Hpki&u`7`2b=mXJ6%0_Qj7WspX`~b;hJ(DM}YwzNA7|8t?fpO*ryq&;_#(FA-|Epn z7^n_M&TKq3Z`+=AJ?MWbEwd{8T6>wlmssmD`gQWtu&#FxzFfI}E3Vh!i><-scmy*C zbq@wjb8C`^OX#HQrb)R=8BsH(+F1Eyo8{W*(6jE}6FkX(UcSCHYQ)&|X*8LrkntlF zg-6MVhC0v_TsPi5bzQHjV;3~Msads6UbB*Of2d?}tNT&^Ts3X}V7Mz!&tTm_{c|vi zdJlq{BPOk%ISg$h{FivDts5>LNEdlulB;l>)-?31lyO}kUcAhced0becVv*Lr%|?| zJN!rRkD5tla_@k%4^QI3M@$lrB#V+RnxTr#9gf*8&WLLnkRLZeweowd-fCV+C??VG zV9eXUB61xURPxu~DxJpU#GlF}@i}5P;nlpyc0Jx4F8MkH(&2X>hF<@bbV7)Kzl4sL zsApQ*^Sb-e1KXk3-53LUg=BXzG90JWpBpE$e;SNuugQegA78bV=|9Txf`2q=Z2|W_UpRi&{E?iv#L*+us$X8#`yiDT-zxA@|YT5`qXFP z%b!wiE9TOTNikj~r|t~dnpSk4Yd&k02FHbzu~;s?R~r;<-Yk4MwR30AEw)?E-Jk3m zo>+&qxHzTY_}?TFjIMC#c$(%fE8;czwO4I--Zw+fc-G}m;(9gdS+d)9zEm6*p7dOD zoB@Qqh34;AoQ6>?k^bT{f2cFK_6BGiwh)0NRL_>emb-mFjb;E(+JS#GHQJ<6o@eY7 zoFRRfeD$SIrs>evp2%x7&=FX(R#>%vi+4Hg#Nu;sL*n~1tBR?kyw^nhxzyBezBN3$ zS;bB(CoO+(z1qxTb$IjPoYlYC^%)FpzXD;LjZZrJTE9iY1_uJ`yA9}n?w}T~Z@o@3c=W?tdONcrL2-fJ10##7(BuYL}g4__`YE-dFjPG#Fg{rz7ryP5KxSe75LHhrpj zExTdu85D?a)%R=5RW%5=QB9q(-T79sV<~3|3@W*eRWp zh!gz7yU2e{slD>x^WyY4b3-1Qt@D)jouapnN*OnhB1j<}EKpq-=RNDvwm6tv{UPc( zD1l9P6ZJ>8drvgeNvlRFR}PaA72-B)VYo zp#Wsb_L^$J(Y_th6`k?QFoe@6Qtx*!S`A8hL4V9ZrOQK9cBTk!zfMOJ_Zt@b7M>U zS5~rR{6wXWZZ7GII){$Sa0!yhiKYb1a6%iZ*c?^uo|R!Mc{YX@TeJTAUzi0)!An_* z&49_`J-^HD3$lOdfOZ=A5A!~iaet)ZsLdza{dk|lVNDGz0JP0#KRv;)Pt0d8fNt!m z3~A>n(6z%!hJ^mUH;Q3M;BELXReK6Rx*Va`*-;&(*NN^hseGPlwj)*ahbZ5E_JipR zb<^9S<{%A;e_Jt;k9h!&NbiFnc(aJmJ-Mg zNd8wJAOsR*7I@eGSq>#UBB<3kUjsFXt7g$GXUv-Lzq0E0=4fCAnOWZZIDQ0r=XE$Z z2ed`rUqA!lpCTN4kW#D(`pP$$L^jFb!rh@-085vbQ6A-g1*$V_U>FRKKSMm{tfThd z*#~F^>4aphlymVfgg84G{`Y_L_{YyPt){2;x4%T1_oZ=Cm8RtUWe32l9P9azx!mfv znd_@#d%?;%%Np2prYIHDQ~f&53kcU4eTh)wHj8Q6eQk}l-~=1K?cKJb_g6({*T@S3=AJ_&T}Ir zIvaRDqV*TpewP9l01XUf)+wqHXP@`aNV10`vL7g60{ey>M@oPIao4Ir7>=Cp;m9m? z`||slOwqG(lJAdO7J;gy?DMy`2_6k{0wlgOMVd;&g(sn)XC&wi%bcG*I04>VIP(Qe zLB_LjV#1h^6$Qu>N7?;v7!^;1BhH=%HF&ewY~zWuETGm{g7qFUV9diP4|cow{Flku z)k&Joh~!f@;YGb)`76m5T7e@JjWdYPhk0b?_s@jM6^A>5&$Lx0|W4N&?*< zj_&<;R2_zZn^8x}efFP06yD@^GTh5)&xgn7;p5-$|C`6hOO_aRfWyF>wUm(ROhuXD zU?CkGr=70hHPH5sVT;|iDC6i{3He70V73`D$SI+wk&z2U+i9u*)Ic02*dXt6PIFxg|W z`Fn?%$>ab!3_gOp!vnJgp4B`vOp*;_@VG7R;w;3DJIPXLSpx$j>z)IUDjVGq6L`w+ zI;B6DV}VtTsZ9s2?4&3sN0a8SiNg_vq7IhtRYl)lK$m{EoxrH}Q$oSWF3Py`n5WAhz7V3yBb z1i0iI0pn4hj)!wZzS>p|CK+E}92@sYG~@AL99y;-?|NwoSkzk(J*L?c^cN(oZQQ){A;jxCRFZ~VgMekA3voc^R{8r%XW8VuUzpqU~XhzP5;^p`?+ zw)#$ixBL1${+w_gSn)|))!qjcdu>0No3)=xVf_ct)GE8q2DNEBA+4`0NO=-07Zdsi zgHizm_8;xld`@zHmRBRU^Mi4{w7Pu}GS$>M=|)Rbh`h5itp7u)ToI z9JQZycS28|>EK@H9%C*!SPY7AnPxh$+xZ83ziw z7-=Tc`o=R`ghkuko*6iS_2FBC7|eVIFo|AabmVGP6|& zV34BCdpMuE{`Emm#m1zSS}x{h6PTNOlR6ucknrxm07Q%dq3a}3&t;=3b1f&AshH@& zk27N*=bZpqHV&()X0CDer=MCltiRuJ08{93_F=83xHNnwlhxXrp9b%|;nNfR{j(09 z?N!atq&;mI8be8#1|C7rUhR&8vZD>EyD1DW^&E;@@ zK^OhMxk+dAS#H86hRRAA`_qep)}~$qk=nQ0^C{hMKPU-SEjQoxP0fqu)kq zRxQL2m!a+9Y)rzobbm353U6z@La+>cai&km zta`gRb4mqO?B9@M6#P@hKTCk zw-{pf@^Y{m8G#B$VA+aoeYScD|KungpHh8-yMyZ@X2>4PR>|BOt7VZ9=;B=+L#&0` zs{5^At8&^c-HO3yDgXXZn65X2ncRKXyUfZnZPK;}Yut77vBS5AnBc2ZP-NRv_l>D~ z$^7{AIGy9!SCO6J7d?9AO%jnw_Om~?+L9f7AkLDE0cjeUs**AuEQGX!qbKREmeT*5 zJp}*zrok&0N07}@D^nykEn*k!DE3x27n+BjVi?>1!f{&3CPX%qNH_)D#pY zzeMNfd;STWFf#mTHc?ryUhg_thKk!k{G;pmBef>E1eq*ve*TvMv61h0GjEqXKC%rXclG{=kp?RilSf#ROZ@!4`0r(ii$ zh1$VLi-Gs|&r5(mr66f2xJT4|=gswEpgNn!O1U5U+!gByu+-c~-f2?RM(m%zcN+Zf zox;MSs%c?ioY32`X9q4+(656>Sm4mjO*jJPJc}Ml!GY+T%Bw?Ht5_<@_$^|%b4wmEd8&sZL>-G8l_7Zi`dosWAPJ2 zG_~@^Nu%AN0!L1(L-~Pa7URw6oTqhgrQDl@i&+^muH#Ol>|}H}TnSFI*|MlF$b#h` zl4pp4MLw#;FD$;oPq)xaenW1Q?|QIG>bo_U{PDx1O$LcilsA=%-wkbIAnPF?zNWP* zM4#Pcr%hed1en#G$82#?mP2Zc`dq*87ekA0mAdJjnR_GULQ6@k*2oS%mjN@~c;$L! z-KQF1-n5~O$z$R>gH_DnhhMd~Y+He$l=|yqgYsouN{1Njp^j%=E|X;?lMcMq-?)p% zDGif6PfZSHwTkC~<)SH=HXPdVUk$wJ+2@Z68^0_9OKMzgZjYzvIfo5J-nvQGcVIW; z;3)8gQ1}4Ll-mBMKQp^9<_U)LNsqJlPy8o#>T`8!0={I!r6#s7cc$)AGmG5}vW#cF zm}JlKBu6zrPEi{@gN!`u^bhKd_eKI_0~hv*YB5E&n(Icg;N$j{Va(%dC#c$GBW?obQ=LGv5zV2KOpIR zy|jynmQRz~m7)gf_wx)Fo29qfsp7i1=6-)`hBQLgSnH%4@i*cL{wD;u=63B`gP+Z^%l}if#zO&l zD}8geJlFHvfG0+g+H=wZQ`1Wb^dNLE7}{{x=Kove599I1 zmF1cM_q0`c{1(3ze^YTnK%01&NVLy7*mk>r0(?dKZTd*;|G{(HfsKH5En)K?14+{T z=4HsDC9;!{8*|VJoP9vIbIhy%(m<@BI!$ZP`!(m6!?`G+JnkRqsd*|{7sWI$AC2Gn!Fi0i;3H8*t;&fw*RPCN9 z;L)$8{|%3}Yoc_ZDp9?Om}+@jp;QpM7zPwfTInpex(8tRrh@qM-hDhYZ?KY7UrIz( z^1gQgGgTU>0_cZY_SxZ#{7;fW2<_9g3`60Lo0>k^LIhp`3K{2f&8|rTKwQN?AWjdp zR__|aTPQ@DUwqDG1Z;9L8#(h|M5j(iSwT2bJszqdiVwO zu>bf{^Sa>Qfr2%P`rm#VCNx1Uq#qo=8C61cb^+&?fxMCn7q3NLrYJ`Ldu4zB;9V)< z)r@y(YPHx1!+K@l{@1uN>MO4T9{uxw;}qYD3?6WCbElg0tCru_?~XvdC#1LBcZhb4 z$NSv_Le2MOBfY5pcdyja0`quD2PIGlg&Mx}Y*C$VMt0TN$c~`EPq_WTntDV;)2RL( zPS5flo*g2D@&oSeomAMKMfe5(pk~7;dpZ5$YYq;Q=VzEhrIY5Uzrf`li6$RQCr!h6 z+m=Bg_A8?O2>AiX&)|bw+WM;I3kTxSaMufXx|(DKQX`+M=N zxbInrdS0=Mdo_#b7k64+gkdl;diU()Ua#SgqrXtT@5)dK>8BuH?ALq|ebyL?RWXt8 z)1JCBOTE68Hi%nb4j{JNw;=ysXa>!@!xI(i6WJTKyfvupk+kHUZ|}bOeTcE_;qm(y zw)w=;CC&gsZh|~xAfcM;LysZ2)Ldc>B2Z_4#M@WblGF=%k0T&ot^E& z0Kv^5|AVJ~npx$upO3G64vz9kHNc#-#l}3{U7j1Mw)d@e2PeQqI<=2z!32!n4(nQB zbP813?97EPDqRnvr=A%~zb5(Pla-XMU30NHl-lt4h6c$w%S_dLBh zF`*UZW&UJiv+bl%UcEey_QL25zY-_jesG3h6um+0>-w}}6~F@-BDYj0;8VeI*f z5&`ZRgfql1SBPXbCMLyFWa7KA%Tv*nh6~)kl`MM6_ki^W7B0;zzlsCmw~VV;i^F$l z#Zm6VbiPYdj~{wB14e>EY+mtWaCEqSxXooThLXG6BRfaqt&n;JD)RV}(O|I<;DyO1tU_VDorPnCZEM--p~4^gM&Z( z&FM*?D4n!owo2|%SeCR< z+@mHOE1SR-e1kOZ>F-}e)()00(dEJ86gkJZ=a3|;2{uKx2Bcr3)n@T_4IwcEwo|?;El8NW|e6+V3&hN@(KJC$- zkf9cXYM$g}?j6azuU>2@7RP1+L+?gc+@Pw~u0?ue@@fc+2?Z!Oyl;{VG!8o!%En&7 z+!o1@Ru?XsDl)+MeNvId=nPBBlku$#xr2aIgD&yuAR0>jJJN(q3DAmfKcG%M`h%c+ zpkV!d_r-kO1|_RwRoRS&S>B1;bUBrp7ptXN7gbDyrn&s2&x`k;clIogw9j+Kd5 z36x{i_6<`*cW|L-$U8Gq)WYSw?U=|S${~WC;pL^hV%{6aKCVz{t>Eh?(amnM=(921 z5SS@Pp6gOye;meXRc~#9cP|#5-IU9q^R2VXcnmcbY+Uv!O`=30SJfRMUHqEaJlw;` zQZdElF>c07no`3Kmoia#TpE+(Kyt-pvIh5KxxGy?GTd*aCJY$$CLE%FqcOFf2}~#{ zTq9R%tQNEc9{cMjNA@bSloHnn7&UO26nr`6+x@a_7YTm-+Dhd)PEFL0cpw^)Mro*C z>r~MBl907I4P8Wyja<;9G4LjNIE%jUWOVK9XQ^CUWetgVj!T_UxoAOSOAM^fB)+p) zONk&m@EVa!EGSt+kM>%`V&~mrscpXb zNz6;&NQxisE~iTJP>v%IM4v)Q#VC3i3I17-yZcG6xrklBU3%+?pAT^a zm%gZF?olpg&fkurWHoGm5RH{_M{K|4bGl09nzuq%OmgHu>5d5V#cN8t+e72y{usvA zi`^CjdFq_VEafEw>io9KC~;8ehl&IWx`6G(4v;?g+fSoqX9l0ON$BaKfpIy7ACOlmKX ztCAa+2NbhR-nz($YE~?pZUFE9R z0n2Bv7_6&-6xo$6IvTk$wiA^^S(=<~^k&#Jj2%AvdA^VE?E|umzU)_{4F+He{QPu@XuVNun)>t^*GAD)%xmPz-?}uY;u$F(^r!bwDzLX zG_t$Oi$1~x+&2*&WTjGOnBkO)YX z7T5hgu1XYc1kTGt$l{c~dWzlF|9YplWbeqe6TA1b8pKcj?%)cYmma+Mt7jZSCF-&Srb3Ok)Nfmb`t z->kP$9R6^}d+Pf7p;$Iu#qc^v9wnBC}yv94V~%BENr4 z2e>?5(XMsUnUsmTBW5Xn&EZieSV+mIh?t~FM$t^n*#vDa_qVAKA0&G+5v}YTY);P& zWDBuR;UGFGY=7h_IMD9Y*ep(X!rs&Qky9H1PQb zze~;W?lP{HmPZo=x{nZFwHL_+YQ?7Onb6Qi(vQ`Dd>#GhadKlaQN{IOLuhj zejB(}_2#JXPt+a;7KFM%x$J40Ogxj;6Dl_w@97j#TVeMf8v-~MV|GIr)e@1!D^F*_ zMAIx5I=ftwlX+Cel1Vp89vGhrfsudefl+Y1q> z9f5@t5~CLo>!1(dZTLQuBgf}Abj4Yr&Wa(N!ugZ z6*pt0BIw2?)Df0EO@nx(?{Cr)paRqKr9Xzde7X7M+dOy|V8OigpcP|pmD8VwcH21! z$7m4I;(lcJI9HYQzSGYvYk4LuE-)rr5n`HK?@N!Ve&y62CV-_sUUNVV(Gagdw#nxm z%kwZ=@XF3Pmi|QRe)P)aLkJ5Lh3F=X8gRlRUlL%Ts|CZGSEL>dWjvb?dBszA&jxG@ zmecMMWwo72c0H=qSShi$cVHO5tL*MNFKTEg`(S^e?jH=u@!^p%Ox3%m)-T<>&4shv zm)@4de;WJ1n0~B$J*EDoR5N2^C;cQuDDn)Ic$RKDo9KM^0tA$rH_Sanp) zWB|4jwJU`}-p$X4^4N(vPeu8ed$b!o@QX3cyKgf~$FXTz6h?Z?8z=JYp>cYQKK`-H zEq?O*>$^w*E8=we5#l{nwhy_izrT`26MIAEGngAQQ!#BhgIuLv7|^PS8A=74%RzPM z)>!E>ilgHjYj+nwNXz}S%4u|5zqGjYZET94jPHtsl@_}mI3V(If+Dh}VK3?zOW(K= zcW}b#m1qQWlwUOe+Ip~Cf9EjSNd6l;wd8K%k=BrJye21pRD=s`d$H)un;# zs$h|U++kFym~N$~EKff?; zb(kVxIe~^h8m?I5aA|ji{X((3Yzm@S)N*CuFudTy{(5{?&m+r@)o&lNaY%Hh>#D1f z3#!M#8utfk$yS6L5~3zyU%|m%nfAN|4&ns*$upC4`lc2%`^|Hm6_g$J>)|h8$;+vAI>zZD}Txo_NV80 zj)NI)VS%F%`RU5(sivLg;44I~93(mekl!J$voeQsF=9Uw5(;Bl+5ffmNIHp+*u#TF z-%9tPgf65|cSwEe(IU+;-ex6#wOt)Lq1SI-ANOqP$i7FfnitG%8=bo&rdgi%WK+I; zNrKnLl0JpeYm9p2+uI^ys{JrOeO|3aJF91vX4l-Qp597(|igb=TFC z_?-PL#@$r#BvTR;4MWm@-_$6#;V|jz9C^~TC_y2N0=fF7j)?S^+gb>0Ahmvr3DsAC z>rTTJBjwi&6Z5>@fJDn&7bbbHDQP(uCR|0oB_o8J;k={T*fbLYR`)cARQzT`hNaJY zIVK1VBnT*U8`Hsx?Df`MLqgfs4zE`N_fD4Q8_C#tq9B^JiRD0WHpTpm-S&KWE-GlN`5#wb2AOS0%G_h5{@#RM@6ciY^`m@dE)s}aZ|d+(+D z2C)bUf_ zKBOw7wf8)}Ki1&M>9YI0@MxerOLVAl+EegRm5u+mf0tW^A^hyXROhO+xAE{%ktLUvyd{X<`Bu~AK8Gxt^2SZnQ71&w%IW`H>7$jl znyP)gHCr`U-Wr5!ygU{mf@Kq6-sGdQVmm5J#M;#I65Dd5@&$W$Qk^#{I*uQ)FF_g3 zi|BhU`kP_bc=dufBn@SbhLfWFh+*6*jGt~VGUMe0?=p07Q>zC?i^}m%a~QT4uDFq6 z{DpZUw1BSc`kv`hWBHjFrVqp%3b6B>n`_uT3XyD&7L&lo!J{-V$aEEyt?G$6Z(jXG{RY?8%uPk z!rXX}R=TeoEr?nciwMJ6$7H7YT`Y$MpUZ(mU}R(o@#fpRsS260m!e{+sPtE+8yL2K zFkT>x(Fw1YamS|NjvT16mBNn-Zx0twF(eo$o(+005w{geS`%0^pi(I{#6L=K_kdS| zgfaxiU(bT`XdjpL`@qg^T2?9v($dmd*6v8!OZUx3ZtYc>e;{FbU&FLQ9kbS*$cxX! z0u|m9e77iG6VJwIDv3WC2QLhcd9*((6fDr;nh(eho}rPBmr4iHSjs zE$_CdW__)hX!)0}genmurI0);XB|ZaDf+FCj#o_xe?Ws93BT)&filMN?Rn|Q2_?MZ z1{giL=H}ao({CT#XUgX#qWQSz5VM5pz1;@j^(}i4Yv@Jt;8_+=>YZNhnc;LPIqhJZ zaN2NmchTan5*d8lF$jcV2LE9RjWkYn#8*!Z=dIbxe0&oWw@azE`$`rsd{&_fymWQ{ zn}zpV<9S8(jtE*Cssq-l>hjMC++XC2Qtp5^Z{bF4-;;TwY|V1>i&4FDtqci}PiEK9 zO0IOM)^|iME_GW3Bu1L2YgO44@vI_54WCdcZ+oc!1%njTu*#&wa~GML<>A3;j zU9pQIRP%jfe9qf})VJKDHx>$9?7+CWaVl2(Ocp0~TW;owey6%L28|A!U(1rb-Uz+2f zlUpAH3x1Mju=F-Ebi>7?Im5E&g`Zd{s7WwA9I6mK1J5c${Qqkuhrc|E*A{@@)&~Zd zz(GUuv#lxgzgBIDrm&uCwU@Lb*a`XG5qtImGE_<*;&{}V56IjHOH`~A1TQRb~p9^FNq@Y-tvZTHM z=+?&YU-G;D?#+}lzRhDNJBe>-NyKT1id=)E7&zm{-+o>dYi*iMEP9CZQ6CBZJq{D9 zJI=vB;TvxJyV5_t8E8vkqcKS1Oa_q))=&LFpim)-zsM}4^nHwZy)jaB+0wFr%VwVQ z_s>V2))rDAW*ir@(^pPgv+CFKd)Gx;o0eMDyivtr0mq|Z zx`BhF6ri~EJEuqp^SlCAg8saDpl}+?e5~{er`4%9%H?YY-zenpm~}MD{`{P|^!sR4 z4;6Jy(Ua*(0nU}1Wa*Ldz@|Js+T9xY*mM;Mv8QSeDc2=FiTfE3WHmV*?cN8d5E6ce z_Ptt>V)YAhUWZ$E#H#1^*9BGTAcoJ3mSp}snD)7hq3R#PQs;5PTwB|!-xiwhbi9vw z`!>D7_r`FjE7h6q4CM<_HfVOmm5F{1R%zMV+E4N~8!kW++Pxv{D$feub?h5I1g*_zvb)WFH|xlOp9}7&NzLM3O8kiI@!Lq^~6OA#;j) zHzUNO5Q47Oj>v{U+zcch8~uyNQh`Jtt=gfBpq2=RUAz1sh0iW(f1n={*d~;~?|MzM zVxuhrvBToLT`2yJmEC2J*b7)5gR#;rFdgMoE!=OJX*r%*8M|UJ?%*9iZe40Nd<$wL z$~#~V&Pc$NkeyVhSy6;jVKL4z^S#Mkt>hUmFg?TtoA2M_KJU-?fYaOhT61=LA;ffw zDl>}VNyuB@QRX;Ihz~%kT@i1*eij*szO?P;;L-Gt7G*4;J#WTSWSM^@*Pp@O+wg}a z3axpvjuz@Vfqm&;vO~*CIH-hxOAM^KM)0SKzFDAE-qOyDI%eOFVCkRuPIqiwwki^FxAMzqMLtX~x9OJ_uo>Y@EP z&xiR421!t5do=f5o6v0YQIssD%!?5;G6nMpx+P*9#F6ZQb2ycHZ-$I<8Kz7(IcVF+ z(j`Y%HNe9;ag2<1T|+Xq+-mw2I_~2X)qEF-Jj}))V0x#eJfSk9vQWTxZ?;<*P@$Gh z5K)P;a0KVgmiZ1_`I2Y+h{xUCQ9r*o@k9AYm{vGW`SiD~#C(sg`0#pnqDOl)qgZ7c z4$eCUEiMeGzyw^w&sj<$JZ)-P#KN;9&6K@NAB~rhHAN*zdvOPD>FZqESUHpfkd@`m zm>r63&OPm#{UXUfYj08gylABB>s<$lsP1kr)JwnnO`k(0al71N=W`J56FYQlY9Hm& z=g`?GQ=TLt9LT7N62^P*h9|4oeq(D+EX;ussw{)hN8bTMI-ds;g|~DHs+;f9KU%3L zvs}J9J=7%=k^}~!+7@N5TeIAzgEsQUAEEsw_`X4{G9;;2QrwSeRuWW``2KWh3P{Cs z@M~0T;N_TcFM?)S7Ll&sqbiYt{$%(RkLFY|LWm7J_PzmCFg`nNQ~xL{8?}e& zQtIwzN)wSvzARk|rnYX6uehL5ZWr(uI}Z|e&?{2M2cycwv9&#I#=GT&=6hvNxSi1s z=VAn-8}!47^`;JVlFzkK;!D0*BJ5b&s8T<{dKVUkQSn1~=;Se?7{PxZ_K z7MR|EIMQ&TuQcxg8VSU*8Zq^yNd%hox9V?Bk#f2HLV?DpD4tY6))okPDoB&TBD-~} zjy9ZH0%suqc)`k~u9hI{h~bHg!Bp+CTD9FJ$>@8z#jAU3qZj)#80CrfOV_Q4-?fWS z8fv@%>H|up=@G2r9Q-D*B>@qA)2Hr4@Ae1;iRJVu2HfGJ{)~JTpP-MT$t)eQ6O;VP z`+TF@v?srg$raIW8&32+sd3OAtYX4ONU%`WQU^OMbRw?bV$l*Eb8{cgUJn^_5a{5k zbS!L$cq}3Q0THw`IZi4o+s)sw6P@(^V@LngJLfz*BMM%I+G+jfWKGC* zBG%Szb?aaEy&J6>5+CjSC{x+?Ss!;F44eN_*Zw*hP5)qHeQCTR#IkbxaBWm7>bgYu zCk!frh%H%$h%GQ^-d}lBoS3gum$Q<*^CMq=d_*|LA}zY*wixwEo`ItIF=&k!(*taYebM zXq5G}zrBkA?E)W~w_=#DrUH97I1Hus!K#eYlbY;2R}Dz?Yb3LoL=B$@2&>nAXw8hDvbf;5He3S`CdIxiRQ(oHVB^h zOs&9J#VZ-X4{{0jw}Z(*IE+_VWncFrVl@ojb{067(ZF}C6Xb~PnaCq!SI{VG)-1ad zxN}$f8KsdC&bP@H>gG?7!v1qGB`h^h8(qS7Tb>l4rphHZVTaP>(V5&-gx_M!V?E{f z4#Un+6C{#_f=r_xijAs5!9%E`RX5o~xE|rTSf#AP6c}S$SFQ+S=DUoT!_9t39*XRi z-JWBoe{4yIY8YWVy3yU`P{r9Wz-* zS5Us!{)aXQ&qfN~oNE!RSiYZwB*?#}_xx2kl}mbQZkz9Q*-rg9BQ3wG3(^b^#_Tr9 zzktHmt3>?svzsRXwb@E|A=3&tLr|op`5RcKxAz5;^Htvb65aKNWfmzv5?DRQ0weSw zj>@lh?jFB=mN0Oyw#`?Eqel)s7eO9J#LMtM5m5!8DV-hlKR;V5nv&WJu+qQl=6(=2 z!TI-uVP5%wK1NsK7i+sH&y-Om9?}J7!7ToGnZfwE|4#?<+OOP#%(xXKK{A%Ue{UV~ zfme5cwm}E_^ct*^_Y0_!X)cn`Ad@gIJGu*ylT{F1)6p@QS?4ElaBy%s2yHfez?_78 zUKIRivf{jq1KEn{V>h4wkl!K4@MCzx=DVM5$NrrH=RYVQWlmi$<<7y`zu!mbkixTi zC)xP_ejW-_2#Q%m?t8RIO$1=qhe>>g^)9+==Y>o7{vE>`nNvbaODPj)CxV2H6yC_X zDu|2QBNxRknSDBKLrxmfI9mXZw*>xb?SxQ@B49!UM-{&IEoZ$r)$*xyR#n~S{}nmg z{~@;g`$+$PspR{=yLIbB8y2KC4T@pv>7IY1k`PL{xE{oONCdBfQ<-r{|6+nfBQ;74 z(@6j%zvBZ3s?3E&KlhU`p(s=9DyGkBGV>AVA0fuM5aNGH+xZ>f%1DJ3?djnnkz@?h z8&IFmj$5torZ@vkgx!I$S4yBz`vK~1S!%u5Q1Tmgcla-P;Rg?p4y|Eyy#B4E19)~m_}qo%}o zp@~P*us5U0D?*X?GW355l31?}B|~$aPSC@lyit`p@M6kFrImjxHm&V1~@*EhrO*wFh(RtIeQT5^ltJIw#WO)$daiDSsH8&n?+r$ z*F@!_+Zusb9qdW(0bSb+ltQy~8}0(k2p0TPdr`3AwdpZWF=2(mY6_*&X^Rg9 z#o_LYIDXf=9OgfpV6{5+EFgaYV#z_vdbi7VU7_THxE+z1FQ-J?rv4%D;$SUKL1(L> z^sbbC|0pnnURWVL7}s<#=Z$&^&CHK3LHS%&{<>Z)nONvB3jEPEzVz+o(#xO7XW_bk z`w8|913!~*?{y^E{uMX!yB~3lm0S2#+mjefR4()AL%#Ba_SyEwc4+O|>n$@Q-(BOT z5Z2$`*YSn&r(Lq=(Js3Cv4i(*3PI4SA$X{yi5@&t?ZZ}ZKHg`%O)8cBDD6wMg$Puw z?B2j$T!>Q3$qTUYWHT}u2N_tZf5)UdHPn!AgPQ0x{=S}&BO;4RT;m;3*8tejb*J4w z_(3U4_k&?a?b1XQEvS$1Vq(}9AOFo6cp&WG`gkk)-n-0t^UfFo0-TqxQK1Uk2s}lR z+ab+-dpJgard*@X^$Nie*+%}uG5 zTr49;#e8ynGekZX=EQ4wjbbAK1p3z)r2F2P2gL&w;jqq=ks=a+)(7i_3xIi}sT&>%$CvHJ>^-C)QVVJpjt|V$1oPBOv=P!T z^PV@q<+kOZmODkLdnb_KbVMc}9+C<{&}?S^fF6k!5N#+-EgYdHK3ZyW6ATu=jFKU% z9_fXqW}tc@8f@4vTKkH>1rPfYw;-@Jbw^Gh<)mk&8K*3J zYNdOTl&3Oa%z-XhA-2Am88f~VneeG>mF%cVo|6iF>1VpAlWVB+I6o43Pd=LT6CT9O;i>TNUt2Bw2S9`>7idX~#!z=szq<@1SsLfE zZ$h05aF#=bG1C6J5dwB=Bia_Z7@A1E3{&tf5N7Oeyu2pqA0)l~sJTZctveGFAfpil znQ_jqQRjP5`~w7Fw5)j6CZO@sEA69zjiI2IlPN<&+Ms|lD;6w8hbbPjpYQ?9-BY%L zx0==!oo|mpc+@y0#T8-lj~1XlrO7xpN$`)D(G&^Sw@2x7H#A)iHc+vX4BD8o^PD3P z%;tT8Sz#ijh|QM3gDmYDsw95bln??Un*bAl^Re8vDV^rDM3>6852kx4iqjJpx3@LC zRtC)sHnf<(#<3IeI$jBo#G>50ecMC$^tb;qCPXKzqg1xhJ^!?P*ti_=b^+Kf z-xAw9ugC@OG^gEEpbWaV>>I5PR7pJjY9eK`-CQd>Y{jQwM3}%~@gbNM-8YW?3vAs6 z%sk$Es;59%VF{jAm>$E5$W-ljYs|@rS05x4y&oQNMhysuu!th0sa^xG-%?~isB)~) zJ7sb%m7=?*q>SO4+@^sf6^>W9tQpP=7{$NO#d(b4LQ)J&GO){*iiKPx6*jFzGgkTY zW@^n@5bb6OctHFgD$p+I3-5VX@_(`S-r-pH|KE5?Wy=gn#%a$aBYQiI?7bqA6%i#n zWUtc>6(KuW2_Yk^h>Wc4Y(?4Pex633>+|`3@8kaG_q*@k?>O$`xUTDP92e(!zhCdy z>-l`Fr?N3UdlSe&-4ja@XNp~3>2lvE3)q^B+=swfRS+5rYi4rK`Oyi|lQ6a7une38 zdG;$5OAwFe?xwm!sZzQlXiT+cGSFHB?L6|lE3(9tG5&baEo%IVr+>tp&Ma$@t6yRe z5}hHR@JoYU36(9i{Cu#O@x)u_{cL#6YFNjaj+!Jot9RBHSLC0GrD`kQ71Cr(m{Lr~ zP73A0sd^Vp#3ZeQ{VZ4oy3*Hk_IL9y2OJOxab*eze2tAH28?i&iKJQTWz08-<{rQK zL0L9e%esE;_jGy>H}Y&L{mf&;mDG&Q0_eJzXPoZl1~#8~cfKF?yHi(gDXR@FmWFB) z(hTFOc1kMJmacwLZQ{B0kuPr5C`1If{Jtqpw@~r|*(Ba+>2UAK54{wZ>+ub=n2R$A zHCF0;OQ2BVL=zsc8q9vrrlk$Ww3{t05@*ZcqRL(=L_h1D;RTA~T7WrcCxF_7vZ3*6 zoh|E)#g_Y3di;yH=hDz>)9u$r{_JxajN<`1Q|8}Hy{unpBn5JiGU4uZ7v)e$!V+dM zIdzt({b&gnA1+~2-#Snf-rZQ{B+L2CtZ161^iD<5LY|%xP8&Jk>1m1L5=|^VqMjOk zNM%yFhE`@eiP=rasK%L#PI)Tuo=4Vl8Ezx212R|yUbc`|L-+v+x2c#slrL*pf&wo+ z`xt#+sYonq+6?ub*3RACNZ@?@8)?StPyCF|_cIzL0GHAAf4Ynsi7=qrQuiJQktgNa zc|5uoJ<%%B^`!FTCtea29O4Sbjs{l$?8WY)0o=iRcx22tn$-FyQshbx)^obcdtl`r zB_H{h8ZTp*0cHVBe$(rKx;dEx0U4_Sxxv@#6#wAu&dFoRAe7aENdj0SI8Iz=FgSoC zx+=#?J3y`w^89%W9P|;|+sD2p#JOZN$3Hj>aN-M;^lhsYM3q%@Z|zgW5w1bYPm*}eC9EyP z`ZhNE^~Zga&d}AN!D~|KZ&fAEj4tn)j->`sK6f#JF|v>4K`uCLBpZ>BymR-Syf*m- z{(@f~{TC)%G0c(XEee8oM@(vvG3FEgg3M)}D;c-b5r)42%(qb>6O)GK_^(0i_*a)O z@q`*=)I*jWk|vo?E&!9HA1Uh2bZmqxI^o;mk0fDy`&nl%H2~e6hoSW#@n3Zx>@5yx zR(S4^1C8Mo&^LfMYP=flF}D3%&#AA1O-RJe3oUJ(hF4<7*4vAc$vjPBuT5g43Qng^ezT;UJV=onpnyf8Pu%rpCj; zGaNAOK|H*F@7Ify&UXeMqGo@gAQ`k?`6`3hvET6+?RWtj1ON7fko~b*;PA5hX2WGq zztzZm)IA*oLv|orgz9<|+;(~D&_Z+nyFs&Ve}C|U`*#jbr8;tN2NvacV`-+c52X`K zW;Yp+-he0c#FvodfXF!eSoqf9E%_vRD~P31V7?pg99M^w9X6 z5Oy#Lv--2v>HDi55_Md=>qBm9&eokxnF)m>{jc_$K#-L-7;^r{1yXh$9FSK>U2_>^ zdFc&$1r8%Y5>V~SgM4UB%zM>>77CGwu`%4ujZs}Rq6gAE^4{FuK2dRK&|PpgKPTz>C(t$LiE&a9w8Zd44T=t;mr9&Osa+np3asha{o<6xK4N zC;Nx55?U_Ex36S3W7lpo32p(UYh8_}@XCCz62~FAy9r)BU~A6}l+4I14~ysWn^r%7 zZ}n^bE#(#RG@D+k=o;0n$SOER);_LGCPZ`Suc~!&A36oKJO!`y)mPpBQDmEN35}8! z8t*)}>O48W1^sW@VJ)lXO|DM`o-?>uz5of_nEiZt6Q;*jl*4mjjtycp4mWk`TVV&z zT764NbU8orU3lb31ULyIt2hq*Gny|u`bGD5x5OQtotuF2DFWRq84<5sBL5wu0($vl zQ>3eG6y!F9!vsP2{gS#|Vm(gGiC9ig(B!EB zg${8opIFf1_2xIDADa6Lc8H*=PWu)plLS)lP_$&P^EY;k*6rS0vE^^?!hL+c5Y?y- z-~c9gKHs?N;z(tg?DDYNYcPI{{DSISgDBm&3L{#`xm;}tja9YBb4vJ~`SkNwC!4nZofn6E#@G-ro9VCU&f{pRXHt)ZjujjnFq%w4y%O|gqM1ga9s z<3f7w4|0ioJ}OUo(1yCPg(_WUWxq3Sxk1-kif=ZPZ|*p#8?F)~6nuyoSD%Bg)bai$ z%DJk~ISJjNA{;M(hvta9a|;JG2=Z(mVau5S3J+(A4H1p4M(=>uO&W-xhgeaxRP zwVMk|;5KEu8=C7efa<4&;i~GW-Xb$+5YfbD=dL(|=ZFG&UQ%gOyg;%SN1yn))8nbZ zveB*_O+vIE)jqTi_%sk7o)#49RX1Fu;&bFvN`#KiAE?vGsn$`KdQwVtqO96|ug3;u zAPyrM4@UR%2&9HZd%U#hLlAa>IYRGG)vF(3Q=gJm9Y~)oen&mA56i?Id1Pk8VzUZVmhi0m^l<9idWWb#M&7~-4& z+?9#%?$h~b%@jQw%J{nJ+>ujYnIX7-R=5EXe5oy-^ErwigdI0gVlw0d(P6H{tD^vw zfY+U3oHDD$q-6I`xJ);ARkH^_w?Uhrea~j3@*=YeLDD==6CqjGRxi?>9LOhn&vJN>gS{2_bm?9ES4rr9W^p;&bBRMe@G3(p+~ z3kO(L@Jwnv%uqb3;@)zpsyNAHKLD!-+d1A~dE7@=8nhW_c(^jhJKEh>#|C{U7ElUu0QN=!6rN{%f_-Qo2I4Y-s_il`^W@85z4Z`Cxq{tjiE;A- z9=b(VFSGenA)?X|j>(~TW5?ALgsenP%LD(8haOT6|DTS;Xr#e_vIws^;^L#2m@74C z(|8?t+rvh^gad`|?@0j%63uP;-cf%WZGwayBd;n46p-Q1yx4AiZd6UZ_L!8_O9FI> z^`iUGOHqq^QMkHdtL30xR3<30{}`=p&e8OZ@=}Ur@%xxmA~!gS_T4%qIkeDIo?TmG z?^LLQk+<0WS;MVlsBp=cDQUzF41g@LKkGGpKissp7!M_0;dGc#H$<{&huj2fK(Jd@(Z8^>jJ)vvy8VRc)pl0r203r+YT($H-^E_Ca&s3ZP%OX7~$0mtP28fvhlLhua z_p)kbv)t-1%I~gSV29I9O&8(MA--=!%&vQQSTTl*_vXUf(opb&N+a{F?|)a?UL ze3$jVQ8YFhU>pI|z2{fhv{UF)o0}sqF~s>L;+es_3j)*Oy8K(S0gqJ~0=@iSz7*bG zzZiDw)8NEl8N1_M>&&VL8Wi&^EvH5@9w>|Cxva2_%jn%0KUh+cdCU`2yM*p-SeDZ1Sg?o%iT zNzzrHbwDA(fEJ;iFEd&cKK3IT9r_WUoj(c2Yof~gKQ7HX2NwZP&COJMJ%Td={me(7 zZ?%?zVgaBDgYzkUlm`t*jxRb+Ac)J@ARQtpzLr{F#W+Ps2N@l+L-(<$FV-5ZvzU(c z=j%TPSS5%18yGXtrVw;t?Y_Qwvm+%3ctKoHqvbwLIPfNFn-&J*FXW|D5Rcp*tzk!d z?_{HAJU!xX43(<@rHF@C#sXtFo30+c#_2Mx9!O%1)W}@a z-T_^c)3iD0I&ndgRlnv2yK7K}=k{;pCxLvPaH^|VX9*UYo{BQx^L#3N-@raPPrrm_ zd(HX%u}hJ=`+5@Z7TL3vNh|*aw6)mLaT6QD-nBmBvAQhi$i*<#&U(%~?wPrQxWNTa z6LiU=S>`PFCzHu%|X1H4WiVLkf7m~ z^Xpe_%U?+NO5Gl*^hHJf%Zn&vWW1AOZ=TFw@E%+dz32CVpQH2MlS-3gT0<-Js&*6@ zwZSZ!Tq(Ox3Y}X4>IFb6WM~B~r_hNOSuW!g>RsuYW&w~C&NOdK&Z?n| zoepVCkOCJRLRKpi=(_k7LqI?m+@AL<9Y(Knn26|pQvFU@Zgg0ps^27f?8KxM(z?iy0t9ZRt z=?ULlkxlO{m&eSx|MVPI%qw*awmi$M1qXx7z#Asz=15aXpuTB?@!I0pGp3|3yo!kt zh4k7-uw$_56aN>xA=5xxUzx4LDb~G>L@~4O@>)*Aa&<=CbCd<@FH)W?kk*1C2;Yc zFV<8knp)I;>qG`oJ9K4aYW~(ADPB2Fe)(1vl#i8|PZV9>vdu%q&mJW~3P_0&{XxkV8ODdi)E?Jmdk8KOs1Y%F?=#qVp>3Inu1ug= zS>8vvwZ@AccnZ&TXDYp_blcseV^XETM|1E)FF;XY@i9@Tn{F23U%D>lWPg$_9(r3* zsuf)ayjSDT@jKwyeHVfW8-z}3Ey`Jw$o^!thY$rS{1w_ixLB;MA&4vvDKOl~D~P3Q z2c)!zl08_;+E%h$xotm4J|e}3NyUING`=&&rwgu}9K+4i#A^Gg@d(*s+eV#`!?ZD- zs>|rGG5zUx?K)Xt;JNYZyQj|Fvs9FX1ty;vaT;tQ^h9(4$Qp|Fb*?QA)u)%Fug3A) zKOEVLC1bC??6sRv!tdpE7R?T?{ZPk_5n<>$OT-iWNP$$UZSRkjbLEZr|7z&MrH=A#;my0+y0Kz5Z=V|gp)nW z=QU=oF87P_Z*7z+=tPzVtRN9;aO@`YFI`59ltYe9T&6J+TJ;Dbf~{`4p)VH6PjnEL zWmo|zN~jE+jt*0LY2wSMmXGtjUT5G_HJ&b>Ru&cj#fGV)6I&krrQ#9x+?qA)m@su> z0#Xh=nQ-|IleM`k3i$67Ou_pe0g&9)sD#=V1?5JZho9-O0iKm?MG@x@^l ztw(`G48yTTKtZ0uCPrXgJrhi$n4w%Q^dWM;VB4-MT?h@|4}LhmZug{=!D@u8U>f2DsFy_|N=xp|VT4g!Ca`Q1 zq-B{EM0BTR(naPww62p!^Rx=F3_M|Gv}jaPy3tkjrmE6N=Yz0i+ilp@PJoyvQY{gI zYx1r7^wmtoEv0gL_kvQ_{ zYmhNlCLqfjtr_Xfx|dySzE$H>UEMAwQ{4-sO(JA0sF1-|eTvsuWOh?49dE!%02Cj_ zJxF@<)fuR=@@mr(GAL0Z(#7W>$Q37$;B+>p2mLeK{_VWb+l*CXfOYhUL|zomM|rpO zNfAb_MDfjNn5KIZi-ye>zNFaAt4})&@@rIw?ox2uT!@pni1eg1IN^sC43f5Zc(1lz#c<Evs98yW`DM9ojo46;V2@dLRNQsm>EIqDc@>b`0U;9B z^*wnB?kq_$1+h-Sg%*prtu7E)qYypeUZ(8?F(8$kQLAAonlT)1^jDnd?tl+oOGjkR zv_d6Ckl4#BxraKVMfMH-C&1P;z2lu7G)nD!idA5}H_b1`lLjLc{T08RKjw=XNzr)c zDw7}qanLm86mErGLX5qB@$~hYdlw8c3N7rE;yGk13QPhDs}%@Zj#5<}aa+wEz%!#|P!E>?sS^ z|G@dr2_LqZF`TLpMt8qLy6f_d+ed~!zv)xJfDR_fN|9@AoB9+1z1W&+bfd^tO ztKN2*`-@*g))sVgt9$@lc*P|Yx8pr^2cfEZ9}ppIrOKZ#%&Yb zg-vc!cP=s2U= z@C1&;XI=;jla8Q%%Q13sh{0D7k~~RgOfNHUk{z(kre^xGBZvbfaQ#EfdYmuNlM$iG zfwPG7339Gm*)^T;z0pM>wb1GXuT473igEzJeR(Zc&3dw1Z}()&W_M0M(pSRKw8)r# zG>kR|E?I&t(8kZa8F4NYy(U)LCqEZ}Py=o6nSi4h5{j-yK|%9p8~=TliY*+Wmsr%E z^Pn@~AdN$t9jd4%2yA?9=0|Aq?qM)UB&DC;cg!Z9o?L*|VlivHEzE-=5QeU(+htnxnbZa9`x;c9(_)AuJGpGUKb(OEqfUq-7 z_fZD3y#;L8vSfAD@XE2dhg+pi8UBH0d4lU*BCQ`0elj2o8gGX95(J zU^KiCLb@f?-Km+S9FVCr!}^}`vbhM_?dt$H&l)dilqj~vx`F@S@n2GQc~t`K;p@Gg&TxYf~67? z+Pok>g)oMBA6@;v(0z6C!T#Pm8)5l~VZC>QgrL&!FL9Oyg!q(gpIrkuW-nWHXhF6m zIdD#abT_eue8AObV?v+)&(7xM>P(9uEa4w{1tBYi#(k`GyPqPKA51RM_8t8V)GP!fpRTIg% ze*3=jl-2I-zt8y?f8k|}WUv%I#b;yeLGVg_hFH3aDtDGNK35KqyI4%t`O=P*%Lxs+ zEE}D}z_>0wgnAiyDu6XtpuH1G3p5~!?&*W@T;QF(di9>4IS8?B)MWsll#G!@dkSgr z&sa{w6QHvJTVgNhIc+(PJbE9xfa4mjTw{h%2b#H=?!ru{k{jSW>=5JGUBoZp!U3V}P7A#BLGfs#`oy!}(lvm^(?>hY;o zi5DN}XX--8+H*2W&6{%5KClKeLHDhhnC{{3f%ZZ`pBZr;ED#3<3;k(GG?q=9s;6LxrA>c0_=nV-}k6MN_ZyCAU`kT)x)Um)7e&M)cMy!TD5AZBcq1_4F#%+7RpXs$l@7rA`%0hIrhUF3(f45*<|)6&UBjQ$h0+E1>bQu*Sfg{ zad3&k_t}h$wzHM)mm!FNU~Fh@rG(StJtSN7_Q@lIB+WvsYcEo(+A-6%S|b`V%0~pt zI|C<~eP?NDM-4BDKJ;0gwx2mph1St`WMn85V*F;+I%6vbK{~8`uS_Es2b|4z)}?Qi zS_o@TJCsg4liRq(e@-x$OAitzF{=8{&UT%eS3mU9m5IIvg?AL+m;t6Dv8ck7O31Fj z;?^&3`ir3%i2E+~p^%u9W-coq_tZGU76Q{M&05#1A*6xWDa@g0NOlXeLQ zLXNsV^g%g?b57l$+^(i_Mz+JJC`#hqxd&UTr(yDtAH*AugReo#g>~pBg03D6>CsoL z?zluSTLjNLBX9qlXc7naZY-KO{`e$fcjOajJJCjXwgJBsd*003)=k_d)i0ntzRL}W zi`-SqHoYm{Z)kg5)l<-uT4;IyO^s2gD~9D5KfTp!_(cX;eg+v`JDtVj>9V*8-XMtC zxgc9-eQXB#1YmCyit(IIY^j-MqIoOs6wggY;A!$b?5X*Vh^3<#D$Y?onRKWCL~P3y zFwC%9_$3DvDO}y@kbjfDzY@S-;yvg`y2ja;p8OjG4syZz@5i*ld$Su8ign6&@$w~*Gy2L???$EfnkcpVOW0d;Y*ehQ~JxIMMo-@A^JK_xAL_bA;Wg4x78ClN>E{Tj2S%E z+V~))i-|7U5X=}G*{zm2ci)1O_0J`D ztCKWMy?aZSqSS7$q&H>8PN)r!a8{g2z<3|x{`1C3sn27}%iZv=LB3uZY2Xicg8hN} z0s}3}hylbe;r=y~fJN%68IB*rZXIrq>oc6O2XsYw%(kBidaBJN@80Qs!86dDQg+PV zfqn}_aW5R7?p;ZLTb_Kp(J!RQ?=k-$};UTh(x^~$6<=)mv9g#oz>%}H07 zA;ejxYF@rk@##~BvB^n%D=RD5+vfc(|w2Z+qo6EjP!e=Q+UVs;ON>(p~V=nb%WiD!6fqF z%e4nwsWaZDI{xVYD@dI6C1WS2hX@G>ad;DjYHgt&m(t*@vpv>_%gq_a)BLE+6Qq0nE0Dhncq?Hn&(@GC65gt0h z{wJvXh7P4dw&Sx9LErt4s8m-tZHJDdQVF&oKA-w;d_D!*PdWzq_isWEbaegoU7!)n zfB^gBoo)#}diimh4D~7mtqNft2d(m;!}s5#-KggemQi8A3rNpLa{o(VID~EV?jiQd zU;H#8Zz?;hggI3}XYlbMbJsF6g%XqISZ?@d){q)BO)$zibd z=JhvKy0rV*%_NxeqPaBke%Z*q)NL;ql7(~Gyba0a{i^g|O#D*6!5|K*GVuwI=L*S5G4*mVbf8~^(0C=Yfj zu;hMHOBTKW&okCDxVPiKY8RZcFi8||Dv`Bo}GswHtn z7m{-2JiUORQ}~bv7c_Hzsw0*ABPE2+kjjR{$~+U3mdgAAfuGOs%~eCa#Rw^36l|WG z_V-$%zEybrVf)@`y7a16$T`bwUP~3IH^!b1^vX&Jg+Ab+(O;4lg9LG=4`C~EnTl7X z?5Q7K*qe-WMGr!-k)!!M`5`b&qBPML*+6AuRPe@ zOw0W-MXK_r`oT)EVj>P%+aoBq=v?MY_Xb^D?89NJI5QBj+npvIcndOucR}IcTe;Wu zA`sZf((Q->{SvfGt|zp~=1&~^d9%;YpF)}&VRzQ28+XGMUMBQYP>Vc}Tp2F03f$iP z^0HIVu}=4sux8fBtZT4rn;_dx!`plFr|-6al-KKY%O<5KznK9W#^K_!{Mm21p;mN5 z-DL@iPRHe8$`)>`C;aw)Pesp;19|{FBT>1%u!T7>+lj9%u$n8NQ^2|Z#{U%k$w?w~ zq#PXS(m5K_oHEQX4}nE9D_xuxWIGf*#A^#xPsD2PPO?F0ZV=cl&6^_{_PlWSAiPl~ z^gF4I&ne=p=i7ha@k)osd8*+3;`NksNZ>6n)wv-CyT3Y>$S6Z${rYs(qhl8DPEtWsDhrBH=}y;vV=xWeMife$ zh@wEDie5QYcta|gKlKr{v&53Z6tGV3A6+FMMcOMb4Z6Iha^d^IlkHCPc?>>$y*~T? zpN%tazi!Da^uf#>+X8Bl3i&@f0LV(Mh2sCBuSb{*y07o2JMiv-<1&w>Z)LLo+?DSe z^In)_WO&5axM712a)d=sv>bo84smV^OZk}?b_|nS-H7|>Eivdp{|o3ftE-}^L>27v_L z*^sgtR{zfZ&E-48e5DRQLNHE;z$^^*`8PwOp68{plU)l;&2X|!RR*U{-T|f7*LUv8 z568IGo?`*}jxi9zB8!PEr)d*pe43Z?so*H9LsmeZjp`!t9= z3ifDvHm9j5JK@gt@K?i(i3#vK=82yrH?#Tj0sq(cPg=F%2|N9@`;K2u?ZN2|d&vn+ zi*!Y@^Y`Yu&)%JA)V}!;B&aYknxz3+5fe3lKV9 z2YHCGL0b8G__Cqj)h46di|>pmx~OLE{T!c!Bu2~IgrB-U0}%?-J2>wx^pOvh+iOjL zG>5=rcN3eQ7%Y`MYq}-3JcFLY7BMov9qYa{ewhM7`SZj&AI?$)1I{fTT}kF?LhH6;snd~8hcedg+!3bVs8~dsQXfn-&>Kz zKYunzpb^uT)q6x$JACTxdHs^_TB~$~fyE;#<(8)r%J4z8i6JHBN{i$eq(|lMR5(rZ zm#Gvh#+fTVk{PBE+f`~+2Z%Ecp@8{PF>bjgORPno{Ou(G zfhECsL}$qNhrHHsb#;2D3Bg$}K|+!9#&wBnch{LIC&tX%*M05f@e~ummiK6TE!m(HNCppYt4DUhJgtLh)2v)L73mnaz$1f=Ra_rX zO61&aHTVI7knNhJ^C}=8b=`Qgr0PXi40f3^M($DuW#ZI)&6nx(^5wLOUFC%B<)*d^ zT|xU^&K(~ZX!prIwpM%9Itxg%9E&=VbVR6!F$vzMVLlOf<@Z9q7CU51`cA{+Pmrg_ z<9!oXIdN)&Lm!pgD2*6@`AkDkBvw(zY~RbYEEQQk0rGW8*{G>;BVx*yhUMjv<`o#% zn{(D{Nzz~t75Gv+uRO2dCOy_^+*>+CaUi|W$I?{$6LLG)D2U~@m5c>$4_y2r^Clg% zOdQg5iXZT-ty-X^|KocZ{kZoB|^fj@xE$`IXig;`dbi`L9q=CvfE!Bo^Z<%1DKzT#k09 zpHT$=iG3?BdWq$Q6EQGJpg3Fj+1|cX&e+nRU|(5jREO zju<75<D6?a@Gur$FFdRci;bP8~B*Cq;jix_JGt3U44f8JyDM zFwM9VzbogxyE#5@X4*DTa(_2f7zb{jfGL0xI#z=4*cPU%ToiWI9eQ4x9_%l&+jeTV z+U-iL7Af~rB~i@QQ-8j`*6e9WqZWe!6Xn+$=n4IbjbAy8re&2+N$S~}hVr;9er0I! znocvM=UE+E1=|ATgWIy9=+1V_CL$OT?68GBHhJO5F4+F!RdrF#ABlOr!>=`7(hM;) zkdcV1DKYsr9yPugq0+AOwf#~xM$3}Z1oo~xQUO&xU#zGEo*@!XXS;j z8!F|aZBIt4HtzY%w1JvoF1{f5yKG;qoiIMW8S0lQ7uRU{iej|#bg3P`^>1k*b~~bZQ!T*hn$rR5YSSZSdXQrJ8vJ$RXQEA@dbq_$& zl$Vtl5(~vH#e-Y9-rr#cRa$U!)hgJ=CaJS@rM**@>r;$COQ(f1c6`$GLA&q%p38gt;8;8GRV4?3*v%%MsL4)fovhH8wt|;?*1&1K_4c3J@E8lqi zc@|Y7zPg+XGvSjFGNBUIhFyzfOhkIY_kw;aHY%XFAZtLfwJ-%Y&!Eh|DPCl0F|nNN z`gi&t+tFul5`8Ez?9bc678eR#1N;o3oTvvw5IDV2qStMuJIsR@+XnJsGDwcIN;ytU z`PHs(oNf{4n2y%{l&pnWclzz`L%-`=#WShTVEa>|n{U>S{Mo zQ9fslJb`%I5Jh*BE+-dMsTuP-+fRAlmB7_S=zn`OuN#vj8`>vn^7QSU5gbN=;?u|u zg;srRw|;2d5c3ROj6yFEgnNDdNQ8PvcZ(ZaYZkK<%K%2E-Ef7jWh$=CF6xgGu_y$? zC02 z)cy6KmWsEnp*};|OqR*dW#UR_pT?*D0DMQWIO84A23 zUvD^X`-2@-Yinh6qu%gX3-HvAj7deT^!|yhKLq~GSelMC4n)R@rJOH|Yb-^J0POsC zm;ZCQeBn8DF)GjB4`!4!S=SGwr1qN{+A#VazQ{x4N;l0$CYmhBEF zq~NK)cSp{hFgE_ypvXL;i>3gRk$TmUJEvMU@zMMS$W&*)zD{Z@W|r+xY}s)`ApO>9 z3o{k^w-c~}Q?_0+DLa)*nMAP7z0M5s0L`K=e&~I)3sa>kFD;NB|G`?OG0S2Y%nny2>i8% zHcRP;b`b7qAC*E0piq!!jE|zN*MG$S$q{4LMn3e6<77EnDuwUK7b1G)ZxEm@}fvgZ#~7 zHG!>zM$L@lYI zifG+dwX#0mrMt=~WJ-skH~fLXAl6~b_L?3+qF5x{=W^}#ot&IB^p`ReCa{q*w1G>X zzObC-^NPMO(8p|5govx)iL;kk{q}5(l!%*1Z{Rvp7sb-9mYR#DZRYRHHCwNDL5VVZ z5GR|fVZY;veAQD0)uQ=_uwbp6?oFjty0z&tT(3?N{R=U0>mDPgl&1_3%}@LJ6>5o3 zgGf7gZbZ-bZj>9AFp?%(#I5Z{32wHE&}46+F-cz1)}zWx5ql!pE|epNlkw-56-nKI)mB_Qc&*PS+H}97Za;8r3O)dY2M-TVNLLH;hl+rw zbtxLl;@!izu?2TtJ)&}j2TBc}VyEnq5dvw|kCP4SeP=_+-rHN%Mjz#NLLEgJQIAeU$u*vAYTlS$bZ zT=D2rg&Y4rQ9+7$2}ggVUktaY8K3Psrx51h?}cN12_Xwr&(~uO1XtKU@n*n#VT4A- z-N`2^G30kqVOgu!i@{Vipd~RHC!#}jADs@(!DNBV<*RtSI5B2-t}-9aKRwQ=8pZkw zEy=fbX2VsR{yP>59;8gt-(0UhIeu*bd+S;-A2BG)ooBT71Y&wuWqkObpB)cL%VU<# zqR)j5cJx~Mlw%s|;6g=X>ya>ppw`GF&3IP@0-TQ>;Aq^H2LWp!o#k}RLF>w148z|* z_Iw72%|O2{N33ISFCGReO^4SD(ufBrNWOC*>%x4nzQl7b*kAt+lX2LfZXp0@k{?eM zM^mt(*W<3eV8!3q^Kq=H_6=M#ka<1zA?2Or6dr8Ir^38y%U?u^X=>EI!81^OTPhtR zfEB?~jDH9Q6Bj|QV=KQ&^%G40!?M8v;~zKJ=RIt)c<8yYRR?+=S5E48FysqR_T#~? zU(v|;0SFK4t(6Y#5dRn8><63-C3?b`E6@n)iHin;4`J%;?|+R5D@uiTcSR4eMUM!R zt~vexU&ua@&zf(2c_bi?{f$t@zmn?Y5NbIZL_PTfCxFZ2nfNVaMO+jLYIGXFBG0!c zIK0mDlF6SsT=kBwt{-68X9YbV6uNbEe~!ovUp1gDa2Sj1{t_vN)D7(fTw>W>_ZG?{ z#^%^?=MT}9nk>GSC4afX!ykyA0YNh{D&28R{Tx44u~`0wW+6ljax5xByH|H$W|s%R zD`+I!+ix=H__t8Rpivix5yp8RbvfV_-!t6kndt-ZpZ_ZkxA#@X)gk|Qln&ma{;@;_ zl6?+sR>~8F><~Wymne^r8YSCU%SS z!VCFGh`CBSXLogiugvSuLu0@2RX}rIRT}Wzo`#VVH+yrn0l=@3#CALml{~kOInfyG z2ZGT)EL6}?U+jet=Ap%|AyDW7J9Glrgj5RY;#p>q)ftF zuqR?RA78NNez5G$is)SB0K)J8ffsmXm{?t1jee{=Uj+G-aT{0q1CR^RdMF~t-{bdN z0O`l8^_y%typ+_}(J(KIdO9YXZsFq3*N!I;=hU$v!I2o!wEe28++`@8f+95i)m6JV zmFe+z)-N`JzIKlC?81Kpbwo)75g+bkm+qD27$(BRvas6yJ;Q!ly`+D?Z9bS%Pb4ac zvtZBygJYAG`kTNw4ueOM)t0WG-`SF zOTv%ye0PR1U=&_=F9CauPc#1il5ZpOP}ui8n3eg*qe%335$(Tved$3tbY zg5ZR-^~1CSDjA2_DzSpGbG@mZY7~p7zA^5I7dHZaeGE0>g7_&8KPe!TUz<18{igi1 zvbiEY+vjq(sLu_BxXbgcB)ZoYMt_9o>C&X#OX8ro>U1iC<{jmYWbxgf7n$6vCWha- zQ;6TOE!r-3Re#wr4dZ^V{hoVAz41ew&{g<_$UMDl2M3wpMR51ckj#Gja~}+3 zh!i5LN(U_Jdt(Z80f`%bG;A`ido@_Yb5J?RpAW7ENCpr4$} z#%e7qgd#=ASr_|b6rJtshrKH&9K5ATvNJZ- zI`rPeZfAW%e(~$&s4Mc`jNvfGFK}n6o{^J|_>{R)QX4Q&@R@NuA+e2?s zr8}RtKN59)cV6S4^~u5M>x}W3h)K7fUXdOqBn4k{SskCqJgZm!n+N8Hd7G61a+a8m zxZ-I*EBzRec*ULjd#AV$w`ncCY0W*w#CR^De2TvOtqY)$4z?T6q_qse)GJ@i4w={_ zV+#Ih(Sf4wFBJU{|H(?N;%8^JwlFtDhJ=;bULX6Mwh$GhI>R*`7bPex53xBbP)sv1 zDmMfcfH^5gi_dDibOd(NxOs7xd4F)03Qm03q1o9me!n^`-`nZOpiD0y>aP94bmJMs z-v41Tqq^$Ez;7oQ@F5&`d}BRRMT(KP?4b%XnI-Xx)7>rnAEVwx-fl{7HA#u5%mrolN3o^n0^$ z{bR~_ULjF1u^F-N*51zS>|nv8+DdJ<$mW-wrpiQFNRl@VH}8o8L{C;=&>;ofIj!N+ z$M)H-;Cp%zKj0^0rpgzkc7+S+eM+oj0L|Js)&K9)bZ~1Z~Z~u zXxUD{{u(Dw3gSJx7X$ybA%#iSIo6jt&z#OTcE>O=QTe&X9wpo_K>5=UI80*Exo@7> z*IYUBFqJwMKPHanP0f5t-Uy<+axGovnyw$}_@?jY%n6S&1?{yh9PdAn1_=LkZc_O9fFiKd!S5&Fq10l$SeMNO^PI&V`p?z*+RlFV zq8Tjb%qcHwO?pbiz@IoJKS-X#rsc?zk|HB`CU4mSPovaXoQy5(DpF@$O1jqy(ks?$ zI~(5n^?!crEELx$xd)H4Q)B4HyATD>t<|QU>-%2pU3vG(ssfNV>D1Fg&+pfNrdt3R zKFQk6fwr%&yswfJ@)uYoqfBqtPfeFKV9cPJ0y)Y>AAj$k??~-^u+_=fdwR5i^hw45 z|3AJ(AbblXNk$Z;L{Pcx-^4a}!|F?Lv2BxtNr-_l8V0!Ydu%>T5m{4t>~U*5x4_*% zziK^$7#%YbUc7()ZI{1wX)_atiHCw^Zk`%l)wc87V*O|yb-Tr{cfrx%K|;lPK_KM= zDsyRxoXKA4>F6sTyYXKmUvytto*)HE7D}tlt(jejXAW z@!`VaE!`P7?1>~WnP3vI5jx%m2E2yXm5hBx(%NBzeBuNM`X!eJ@f*r-o?VwS>!vgV z@&cjsZA{X(XMkAgG-<*nR!V*%)9w`1?EXo$sz0Y8T`(uV0#t zob5{0N)IM#=uh^a`)1s}bpPe`k@t1JdD1Syn(-f(0!H@!)f-Xgi8&}^PnIZYp*|R%z4O5h=K=;dY84j&#ELElyUh+T;ya> zHZWL`4fWQY3Fk7}#AT4TtDC*N_VbJCO0Pydg5b^_To^fi#k;=LLslN_n_8&>-NZW7 z;;y&_2A?#BuwhG+foUq*->(rBTWj9Gx&2$bfXVaR-Ix+XgXrx3_tK|S^ndESnhU!( zNBkbZ*5Xqgi33vF_T3MtI#R15;|oF>J{9E8>k=F18;&;F7~FO9j`g512q-2suoyVd z?a6q-x!mF%Dbx@J$9RZLJ?UyYgN31TiXg$1>|1*E;!MtL`kXfLJ@2=u>{z#GOkyFY z42>T_IPG@#B*O(b#9nd)oVHv)a3@74KrglW&9H;x_d$1@_NMZyQxNfUjsTYH{0CLv zV+eiYowA?Fa5<1w-Q%LuA;iUq%`AHUQ4C#>(5|8j*FE=-l#kJJ{s7h~T{1&Fb@+{Q zO|2i$zfp(gx*ym4NvTeSAE0JTAjk_ieX>nMPwmzvbftX*J#}Dm*tUaj3O+=AG!G+E zgKuh?!!YD0=2KBovxlslqa~!{4$RC4CCd0C_{>JbX9(yYp{0E!W+68g9yj1i!&~zB ze=0oqP$=|=TD|abE&%7?LurwdU%;n?--sG-(K)Q6YT%FWGDi8Jg=+-lS)dWA@9#g4 zjM>8^B;+3YaRam9HFyEg$rU6_AH$ws!Adr3j(m0k{h%awd_A=tkjJ2jgVlx3*$; zbaaJZHtRrPO8NK@Ju_rgrYmy%&Hui1;@`m;>n3}2_F!G<#7ERjl*7sY3+{lL_WyM~ zCWxWYR~@b87Me0d4|E@{cEg7Wh)xRVsZ)nbAWY%L`eF_w>78AjX18dZbM8RgIPd#s z$MY`406>e$b)(aJoh z2rDcNGycEfQL#EzE1^N>2ETrUj3c<@mcalosT#e z-LSQ?)wJc=^A#~Z&=VFkv=(`eMkcnS<7(t$dqStmGhtzQ z$n&>?q$Js&3$OH5#Kk=xi0c#>ge{MpMaRyNFWHaQU<D4&!#6~^wns|*N*Ln6qKE&e*| zW&^~uH(8R-4qx9pBE90sub+eg4JylOh_5g`4#u=|FhN3QLkXWGbsHbm+D)~_vCJz2 z1$JHG@>KM**OS|BzdP2C4$^V^l1MN)bm?`t#yta zG+aKsT;9JfNxr^8wKEQIdf$M%kMY;{Z?0Q3Yu;bG+KOTD?2RVrKk6fMl>LhS+^#OknH{pCFChkfX7;R z0*7G!3FIS(02}U0*R!I0y>WCnan$wp);~ioL`cxZt9wa@$ZQOtW#o0~A@joWdG4?1 zmZk}~-0}Iv(Gnwdvz8Pny3fEWQv1C%uwEg@niPk$Sj8JF@gHXDS1#wZ0TY@_loP=) zo~OT8+9(^5lT?|1uMF$LC5pu;mu^LwRs%ScJbPZHNB@~pp8iI*1+cmwd&2xigN=&H z04#6$8k^p-@PB$W51I2JD`gl2xsC2PhDSeFaz7X}9b!y1)V}pU-^@4=wJhxr1hI+? ztYU+&ow-1@&@)_$3HtPdttq!__49W((d6L7<^D+WK-41$zis3Q^Zg z7>AbVOw$CaC4iDTw|GPMc(RQ9BKhD|a3};rN{eWkw#d~h1dX7VWlI)tTBrzn7h)3>EZGw zM!K+X;a7t^MH5`AwYf6i@m!|u{kPP|v>3L(cE1%a$ROU?6Y$4gdqydU4gg#5QN{!i z)nQzR9ZKl^+3C^xgc%TcuK-8sEf6MeNKONx!OHnv+?H%YE%1pa!t$NnUkP9ZBCR=- z49%IMK{1J(yD=!7u{jE3{fewX{qHD|))S3!2%>jdrSt%h5aosOXtSYU#p>yT^J4WI z2ntPO1dh&4phs6(KIlzw0l9J7L=2gb$HRr5oyxGM-RFG`B|pb3-E4mdk30lVqQlrs zL-)f`LMxCM-d9_Xp-oRalVG&8BR-Ruea^D6c6KbttErTUd#ASs?wBuY<`q(xAH!aG zK>$O+?-F(MQ@~PxHSB>sD6_r*7hrF>rfW#5G*EaZSebOcW?O#V;dx3K@^6NvyLS}~Xt$(-xYWZKx=A*&77kMdi{<#$3 zSHm!pe6>ZgzY_*s$Pm4gJHX}>aI?#ydS|Aa2oWAOA$0QKmMW4XD;c*8Ul}HqHBYT9 zT#(j=bhlNDf!ep^52=Dqz*lUP6;Ww$758ob0lO1Wj0wQO?D{Uf?P$IV84PdK+kqR) z13JEoa*1}Tp;uAa);~9V`ME_O7F|vE`MAPD$u>+=hUY0Z33kzr(#}$uriQ6tdg}3D z4fO*$a^;qaG^Kvv03mpa89x8~-f}b79fJ7Iw;aW2FBKZQ5Mt`W>07?oH%*6eCi9?W zOKxAlUt3dWLJ7r`*+%X8SYtULKw=wX86LMXM3yBT9m9vE1}Ta(0oOeDoM1F8-h{B? z<|3?)wtYY+uc)XPg+XQOnLKl8(W?(Q>>$6e5h$X^B3Ph^*91)VNqbN@MqNBRw^2+d zE_}8aCg~cX76Qf+Jc7jMH_PVUC&xx%j)N9mgIKYKz@Yx2qGS^9wJkx6=odg}E5h|e z=D;-8uCI+ZgphKi2seY}9U6n|LdEPg+X;zB?$5e5rkFOt=(d{D{v!<@s9Uv0#sXys zm51}^gxllTwVgVj9yG}$feg}6wv*PP5FkWdNef2kI#Q^jxf9r;f-KKG-OkNKHt*!G^)>ak67Pg zKk#x&!~70GKitSniw~l#9Cb5UlGB$c>oKXR?d4MzGQXN|q*R*qdOZAoBG?s?76xH! z1-S@vsW)LkBjK+Y=$OX0_Pz@T|J)+d!i;UQNV?;F7y>qOMI6Z}lFbh}Z)h_4DzxI5kjv86ZeWnmNU7L3*m24%!;Y(FpX8|;+%L`w;XVKk9Q3fk`UMk`;RB2{D%qt zeWXT^$cvZ2nHFi*%SUTHdTMJ0;ugvWBdp<|d2SoqM?fg1O9*ysW3P0-W3t8AbuvbR z6q|9?EUyDrQf(CUFoZWGAMr8DCxxo!yc4at`R>+D`{_K@L~F^|v_cJQ@L!OLx>Kai zlG`sJlPkKz9NYM|0H)TEkZkX(jm?0~D4Bg%wXoNnrNoke5vXnx$=pW7=KVj~SB7de zcq8EsLiea9@mh_-;?bGrir-Y|pF4<5Y$PF%rXwKk;b9ucc`+Eqq>%ESSO)x^FGAL9 zgU5_=Z^HUOiLUTSItLg+6~N^ysN^M-jCL0JkS?T#w^?DPD3^<`JQ)0f+mav`2Hv#p z+r&SikOn`)ylHfXrA)-c%cktn?+K03^cco-?b@r}avFto`SR{DId+iQBO5hZI#n>; zhtS;x8f{?>wX85WscGu|gjn{GMzF`Vk@cY!elF~L0YZcQpIm${TJt_st1tPwMm0Sh zLyWn>`wxK!-ZODslB?3seZo`S2t{$2S03vLn{*-MQeI>V#GtPSWJ$^uT$cb0`=+Ek zNOqozKK9V^c3g+@Xt;)VJS#V66&h0PC%7vw`&E)7%{noaq<^Va+4}laE}#dAhG!?= zPo$H40Cs<}3?AGwSC;N2O*eUlJI+5TD)Ij+$+Id$D%g{rM`EQH3|-%3&OIn@QB4stT!I*tDE5{2x^g!PBk52no z=V)`i9UqK#{SI@&Q_oi|dUAa>($`zfa^IPuf3L@{Oz6OiGHyiM6Q01nbz0#j70FR` zn8zJNs$KI}_lVDkCXuv2hPmAvldJG~f?d&wN7@zaobZveV7j%E`PRQ(88VXM%Ed6E zF?7%cNdxc`gixqCRo&h`^^sVn1!R_0CJAxr&sL zouRQ${zEIle}xHl)p}-$xOUN}E%BDIjkO9q4<3A1#j*1?kV3u72%ww-NU$i7_As~? z&bTGpRE+LrVU6@be~CA(e5i1$@L|PPoqQ`BAI8age#82u!R0>WRk#gkArXMKjq!x- z_qN_t)8fYU{YGjohlx1UE?8wDV$^eF6*k{gKha@KwZS|igos(PjE~~};h+3&ir-_Q zNhLwrT5io;D6>^id#8)Uf2F;T+lf)}$I;Gk;orohQ3;4!_>|Kg8hq{ytU^nOURU_1 z|2|W#z91+^@|OrTg^dcOMvhv^`94F{O5%z|8HZ+JOmq<+n#$jK?#^&%knqFf)JC@AHtXDwJR2+ z(*$g}tv7i9Ek>3JP%3rD=J^>JKJtk0i;TRPnwGXTcRr|x`{w_a35to#1hu@|_`ha? zBDJa)nzmB^gZ3kbO(IC-Q}p^5l0UG;KPbipWX{O)v!$xP58UsS6967a*eNRc_o4nk zTbubF9)vUY?bJSUm-ZND3Ft-l>a0Hz%KgY*fj4jsoBi*3~-&o#YR&H-;>vyWm$Tp8;6~WI!3b<@^;5 z{H3Wt{ORYk$okT8KH(=oF~crb`HCTbuz`I-E{Xu7m+UoBJY1{j8-fL0Dt z!Jl&+!i13(Qg|N2VNDBOMRV434!p2njer(M{Iz@B2ywQR@SBoDjuSEHJUVCFUQ!MU z3trSdlkb1b^hUM$(H-3zLc&(_W(jyrpvp~)7WOp)9S{aE1lnPm>3DiVY1*H9F_!6r zB(PgGC=Y~x5;D9q{;}LcYU_3Qg8)VpqLt>h125l-?2ZF_|6`{3_1&e_k#96W@ZQ{= zLWxeC@#&2O&k%YYUZ$ty*U_Ap6s3}RNJgqbQQ8K2WQX}K4b#39A_%JrijB=#=y3t? zS@LZ}pCj0r5OeB@$%Gui6uxh@1+YJ<#}{S}6Mf9s0kbOXnyZBr1#}>ZjCw1S=v)M) zvH}t`QOomS)r9k+u%Xs|hPA+R=j;8a~2r7qBYrQjAA~R;Y@=C z2+(*?s?*r(2y!`eM>9S5_cZBQDxN83$-41YS8?2_T+4fwI-oz(rtAxA(bLx1>uPy+EdB7kjmqKynqhI>u7t;$MlrQ+&UM|v z@b7(}1Ev>?*Ca*+T@0avBg@Oqj*bpF20F}qG*WOV4Y8pe>OTqLCSg%+pD~h?IaDsz zQdA4H`XYh%KIs}3lhQP=S;zJ7_e)O?NmkFj1mW=1#KnOuNsUCwNGkEPJXV87+^oCD zd)Id=0D}MVZHxqv>mOv-21|?dUNid~T_XTpzikX{G`+(6s&6rj$6y`7RKAk3HJ>-X zlyNKtUZ5}B0}Gs&3>;if7zxE(Xlo6!~jupyimmn`CORyQ-n7ax>Hm89H(~5$s7!Cp|wT ze^xr1X*mA67a5U4%t*P}?x?}v;=Is}0%m|Xwx3I~sg(vdcvf{-~3xzo8mMfIjV7vjH6QW?eUfrH{x z@PIG6E^22t70o&I0WyUWS0(F!>x-%{s-hcxLwUX?$A`ilTi9Pd!le~^4$HF&tMVAAfoCd&`dpCkdFHznGFWW>ilDQm zgzJC}87;E$(jkq?DuhBu+j8b=)qXILUoiWSeg!kVE^|*o(xvtBhj_WaS zDz-31&Xi-tM}_~tJ};%ehT?uxeX?#V8ioMj+Hmo20%`ggV4^e@{kNr~W~IVEfFWlH zP`HsNPzV`Uvj>VD(l`RK3rb-Mfhw}a-g}aOS#y;70>BhODHsA}4Dx6V0m15IKE)G zqh{LDk$T_q%z{eHL?3;eF^EN4qEUni<=`y<9rHjfEb32a_uU0I8Bwt3|CsgWyX9DE zxE16eeV58p>ov z5ybim!wtItv#-{lTSM_t7a^b&E`eX!cNt*SVZ-gxP+@Z0hpE!2AIg;DtRI(`w-T*N zR_Dn_IbTRG>q_NkW4|L8gD4nDB)AVLERaw7raj1-gU-xxzDtdOmz0oO07%LM_s%WZ zfH?Y3uXd*)j|&~soC`fsDJHeh=b3@(Jsr=nMU`=;2b-*H6Kjcj$ntY z2(*w(`M(OM#K-IJy{-ta&j3f7VIdISHy`S4Ggro$U&pY zh~kNbdu5mi`~<|phzquUeb+B{`_2jGtps*J%_#oD$mq!!?SU+4Dy5~ISEXVA`MDN(z{fttzq^QW=0`zkQ#-i`FL zF^Rp-Q)%x0(slg_q^RKkIrWhaKdTmrocgW6$C1x^1?Kb*SGH|&lJ)3y-*r7DM-5%I zB;bg?aiCz=n?Q-oGC>+G_9S9z08b3KVxf>TLsDZW^r9w511yf9?@;$mShp7PywnL> z*~nh?4Qp4{Eu4H1wqSt_zXqSt_7e0|qCGK->LW`ZGO}O{E_;L={7u2wFR!1&Dn~I5 zn{^NdWRys29>a{GT!p~IF-shym<;e9l9#NivD3#Jr(*JwhzP89C{xZXCp3uWJILvQ z5j)T=m(J1aR8W40Aw5_%(?fTfepuF>&S>g`!K`vUdJF7CAXXU%{H@55&Sx&a(wnRk zcYmW9(lixF7S^8N`vXZ#NVPaVSd!`X_hQ*D=hS@*pnWEl2b^NzV6lKa@gXqAOB&7e z*ItKo`t6*F4F&>m+W2NTRZk+DEs>v%29}{cv2U5Rd1?iE=?AW|z#y~}j49MXsJa3> z(B`g?Q49p5PZXAzHMmhjE0r=S%Mn!IHM)0?g=?Pvbg z&F>PqvTLrsyPb9SSyr}Am_r|sFMTx@n&pON4--4_D_oQpTz)zzF(@K+Nx%Pz!X-rS z3%M61l23$hB;a#|^qIF{ACFsr9^Y+%I-QJBx@o$@X)-@ zcZObN<)J6Y1%fa=I9aU{_T0S(6+Dc(F<^`s2n}RN5>!Ysws^$k1?mL?U*7zx>0AFc z#s!3ubXf@+EG$HOoRFs-$m9pi_h3|gMZ3He0~s&4F7s8;h%Lv*`yTFP?!n>6?kOOx z4KHnMm=O2AsHxN@$|dDkEnJUHN$svf<_2at!>CfMSnCxu<&Mzat4Z!Wfy5)6vNG`DLU>>iLVWoR9Xff%U+L8Y0QpV@455ipW61^0rzwE)2aG0D;mLXf8BnXm6$zg{GpF@c!G>X?|1FO+wRzfKHved6_EO}q8be$KNyky{f^@o z;4P=dJ%qqXa)_1F7`QP`$D~VU@jfXKX}9{im$YU!u9Z&S>4f#})1*Rj87qSpA2Yxu zO3+kmI63b(lS4bLoN*s$OJIU6aeC6h(eI1ScYz;+L9&MRLLRulML4PG7@uhb3v>c4<@SUior_j$wZBWm9iWft+%X%J&+Mdv8WdGR9`!x z%Ug`S$DplPnwyIlEfItb(thBM``aP=mYkbuX4hK``lEWZU$qCxy|DePm>W4!WtAOc z%uNueP@6<1rHZ<3|G4qjTEq9GAG9&k#)@pjm|mWC$EG_82g;BsbVJ4n`}#178@M_C zy=pzNQUlqc;Q}-CDvY6%kcTD9pc&P9>2;!!)IVGRr?1j<0)!+mcuEUX?e3^Ws6?$> zY`@QG+&6dw%pSFy>tK`-xXpe99)LA&07 zO#%>zQRd;Qgc`vF%+H7x3<}Xs`ac{{4LM+v>^Y1SAZ(&A0}jH?`f3+?466}%6Rktp zGI>;iL`;fd0K^t`;(Kd8W*J$ zD>NeXQ578PId)tIN1yp_m4pOJqhhs;|B%=QXvgpiYZg>Py@(Q@`+98a7q2X2s#eGm zT#I*B(0Qn+TIAc?z8(yg?aiu>) z_+>vy=+n^7Br=%=sOyE|-zXu(?W3L5c*w{BXu>Yqe<7XX;?7@&X|6@=e!nA4@a9MY z^@>`S_E*V`f^6l)Ycdb}%HHc-ec323I~Wc&UZgYL`}$EsQN_MI-+^hkr@rcmhatNSVpZ#&Tvzato%{0U&*_Ix2g3&1faqtJ|6vgQ_JjE?h&ObA2KO=7 zImH>yvrbnZN6H?CUj_liN`KByq@V71|I-_PAOMIhN3v1=uTC`X9vuQ|T*y0JT3SNM zmkf2N@nH4_>AmVG$-4r*m{T$ACCH0vQza$y6ZfsQ#&^T1hSFia%%% zwr}bao2~D_FnsfH*E(Dd@vO1hXOYfoALa!|H-Lhoh{N|%s^O({G~t7iTNT|66$?%G zA7e!BC6dlW&l}IoAw>j7(^n(dkIpm#6cqwzYpqnkcC0kiLwIm`AXeN~4+x=>z?Foe zSGdT#JKSmi`@ZD2QA&XQJmF*{hkYriaarBKr{4E81CjtUHnxNwIoxlubepI`XV$5T z8`)c+o`)gFbAyF%```C28H2^=qvJyoz{|5zHh;Mtj03NL+bRa5KlSxRCMFA)sXQfW zNDRKt8mFEN#LFBuqAjx_TTqrfb^E|?a(p&zJ1**uAO)4SNVus6aDAxa`slCv8k z$v67qjy39Gqt}5Lvqs@S6xqwt@%WrJ19U*rGY-C|g?60sNG*JWCl|8X+g-i!X7^j-K#i>l z91Mi>T?%TpNO&v=ClNTfYP2TX(BMNeZnXBPAYbcMN(=2DR6ykzhFTcVbg>6oK|Ba6 z;Z5Y}&aQqNJrOVAMpxPnFx4VKrUzn zw9h-B$}rtMgsr!LY{PYVGx*ICq0!LBI1ClI0gAKv?E3Lb%Iq}Rjx0n9ibKZNE8kQ8 z&d7|`tfm*E-pZ$lPuJ65Oe`$@bq=6eJL9w(wXB{)>MexyTv*!?i*Kok3fJuk;V^dC z_M@V|`~K9^PdJo-{sIWThgNr3HT0myqgD$PuSrDz;SRht2k>>&D3b`SIHA+_JsMCH zs_R{H-(Cn#9HW5v4IE@sIbIQeB$>1^(}IH+*>=lm;oO(Erv!`dG{oLU2+6AigVYn! zJLO~;;H_%CHNHCam-B)RRWCe+XV?Xxagpa*+ z`SwKC!xSe$f&?+K1C4H+w524~p4m68B2CJX$b|`@pv-R0^@dOi*0i+XJO*Y|en~Ll zjca6#p+8dZc%wEqGbs#Y8Fd^QgB$2Aw+$*&K@-@PAe01qq4%EjF4O06*@X8Ofv&2_ z;PN3|A76$@l=K2rZ)o>6f-qw=VaLjrpaX_DE1~n+C!@6~Po19dDzz7_WwK!^i)`VH0NksT^A0THv6}SIZ+v;k6=}-L0qr=SB3Jwdv5y3%i^mSQ(?aLTP{?bRIOh zs=K;X`9)c&&#qJ}`zChlKfa(bkgLG7yAQscB3&$B?mQ~TKvLW_iV_UFaJd-bA;e6b z;N#tADZ0lwZNTRBMH%{ptsB=j~sdeKV;PK-uUNCX+XVZ?lN2vTN-J0u@p zP(aQ!uznUXx(gjdS1bo z>s~33(}MKNiFgcsenG&o1*-M?y`E=Cs7Vw;f=_8FrLfPf()%r0kF0^ zlFt@_;to8iQaXk9j}r-^ugd__;Hln)ccHt-sV!WB&!%ZJ(wMr>s&|VmfpaQ*K|QIP z{Mw4R={-}a(XA&Jo23&sEEsII!;yB*g@=$UEVVJYqCTu7;k+?vB`an&h1E+hwf7Qo zYgawNpIe0?q(V}8pMwsx#M^XPz-mM+3EXjD0|lhWNKK*u zOrlqfcGkuPFY{z`nTCUp@M&bzj$cIK>cpmIVZQWt%FY1dm8Sh^&){=|p7x@$%S{+7 z3h;BEQ(uC(Ab^fF1U2C3+tk4auh>E!2=xZv{JKG1WZ9tL&5} zMis7s4oOCpo0_N=X-QVJ*lLpAVaUsE z<`F-|H@r+F5$Hl)J_$C-V(6IZ9m54Cy9eTiFANo+c9M}~?eumCz%u?g%7FJu1u4hD zH!(9_Uki2g48R&Kf`zp52zP9{DOegdQg&f7W@QOs9R`IIy#%hA7q6`Gzkb+%h^trXxBT{xP4s@JD`yY>4?{aIu-D?d% zO{@V2y~2iM`T$rRcB>c6-$7r#2%*~k##RnfRgurUV-P*4$2LNm$j_hi)gG z^1oDi*&z|!c?(+C33k4X+1DAo)tlwRv?i_il_?zZ?jt5+NJ0R*a6*5YXD`|PALTTG zlIKYKI)b^-2($Lkh>RPHEB8gKz%WWi824w3#R)Z)a*YjtwsVjtM0#7rL*V_XoP20`j7F36O%lBF`SS9nai#J0A`sn&LZEF7F{*q-R7b!F{!a+}Bbv7*_qCU*Bs0{Ktbx_L&y zS82L}o>kfUYv7ia-F28ap@=bT-vt?VuAMz#>#<-G(+d~$-9nGa!a|Sr6>&6ciRXM5 z(Ka5Q$B2jmDm8|zxlyE?11Y2Kt8-*6)?=mb`qSQynBm!3Lo{OrFU~cbXs5MLW^zwj zX4Bt*!+bV?^b;2*unR7e-LC6sv*ozoT#WLihGva6mGKRTJJ|MZwqEO&*UF#Sh-~_(~RR4i$}hq`#P1vbetcw zMiMx8NvNq$C78P&JkUd*qr>S6-O&hV*Rvnf?lwq6|h7zlJNtmzWg2tb1NqG*@*9a6|ZOo59;Iu427yaPuyC#lq;0 zn%}{J`+UT_uMq)3J_K}0v+Y5j$cSUNE|qF9M!fUDcNi0APmtj={N*eifJOQeicRU6 zW^s-UZG5hfZ~mC#g-$jbj(ddcSELEPTf%;jE_^Drz0gCaQ8o{Ly0forG^L z2(@y>O~dqy#%B7lDwNr?sk*td|*HAG>o9E3FIAKv1t^h;*#^+eS2X{60up%l=S6oWSQeb zfJt#SGh0K%r1ZC;(_kDa`o)yS-9Fh#JEvFpU_|!85hda6Y^4kp&C^M}HQwB+k`tcZ(s`c{V+vB-n^GZU2p4XvH@ek@dC4=#i1;zu z3y@f0`bwaU?n#D`zJxr%mwrR-j+FK*;wesbo1(-Zc~Jc)WiCKm*GOy7d$@#(erGfrPnskm zAc?}1Q(P58ji#`b`+jo(P-w8_fc@DUO*8K+{P_~TF!ILhwA%vj~&7L55M3SaN-R&VoKEs;9);^Yq6AGstG*jXFH6CmyQNxN=Kj zk|50dNC;6|iMjoyCCx=muLu50UZD?+F1A64(RbU|^N^y8lq8T0ZlO}{Ugu4TuJRxegACnO=1okvf z8An96BU1Tyq+#gFJO6@3S{0=&XusVx0%(lN(Uxfpc$!khOvISxUg{!*I6J8&mDnjR zIN?|IZD2CzV#`z_#m&7PqGSOxgPKzlj4<;$tvFc*a-Nv7F@sG4h zS7lD3Ym0T=qhO?u15r>O%DbAY<| zq0N!#Z_=Sws`rDPl5{RUpN2C;x5mUV$96bq_Z%tt{IvP;^-lcB_SLaqw)u-L zK^=|l4I&R`KThBYA0eW8bsz0M+ewY`LwW9vxbTVt|9VJXyNBqL`q@iDcl9xOwZfy0 z=n(rdLu7hmvtKYA?IcWs7qrPJY` zOxMYV-@Zkwd(R*p^^ta%@tHs;-Y39SlqyUQx>sDV_rTCYg&^!&r34r1?O+xUA*Fss zW6VtoNdJmViZ>q$e;I%!hDC>^nd5u9>#OA;m1XGEYhsdThcHBo4V7`4SN`OJU@+uJ zvYD@>TZ!Ulzls0C(CPd`SMo(HWo{Mj*JcF4h@>?vhU?1MjJ3CUTvyGJmC=5MqV2|4 z+E>4%@A?I>#0W)63s7L^N*@~DDh544KxjY@2M#ZD8*!>K^8H>K=S2seo7{2WQ<2^< ziZ-pIcybl9l?1<_$I(Yyt0BDJNMMa86;m0^Ahh_dmi&2R|IgUDr&u|!J#yt7f=JKa z{5Nule-U0P`8c?nMFN6}7_oCP?>}7rh?xr>%~oE6U*6E-z^6si^xYk~Qv39I+8>R% zh&rPm{RN0$pzl`g0nX0sW#sQNzSWeyJcHg=<#YT%VF5ozvZuz{)b{XTskPvF$D@!x zg`po>7NvjfOa!fgK^k*X)>Q0bV0-L2bRu-MN2O&npZQ30QO@7Z4_bj%|5D`nfS$uO z@?f^@yMK6Un+~Yc)5kexy{^RaRLz%ec`w1|SK;QbHEv(F`+OPF7Hb69Q!jTA-^KIq z`#suvM1j)9?LJAr6xiW+_CA-{>6e~kvdX2?x5LGCp9a6m-zY1MNL#0i2x>o6u5hV+ zMgm2jMfDqDE8bG!4r)7ZNc8(vR5E@6J@nLqOzS`ohi^@msJvVVH&(vrq}{4CRoHiQ zkN{n6IjNHLkKc_lx8ubqza2E@B<7^l`!|e~p_J=!`mv0nU1Rf7yhIy^37auEYK!sD zSHzTf0b82t%ez8W16h%N0nPK>fPY<3XUWlkl1&G4 ztb~UvJRV(v{Bnet=TU`E9*I*4?-hHqp{reHr?~gr;Rgldy z+1n`T$dIu9vJ~!o(Kg}>|42sQ${_EBj3KzGNCcieDO{-NvATaFvha{4@f<{T;gOsz zZeS@C&4Wf$f9iqt1-}q+7+2 zw_^G6|E3cWV}vS_hkp8>XZX*Woy~+RcKb)0^FJ=4|14r!c(}2XRqAHIXhDMC$+QJL z5g8_b8Dk*u4bTAozNeURhZd%CF`yZcYxY8CFn_P*+TeNgGa88Yysg!4Gv4>_9P>5+ zRX{U_Ny+H7>z7Uf?OMAa2-@at{q6|1gSuw>8zo=Xg$%yD@x1p9f|zE$d=nIhXMl06 z%$E%)?lF)@B>ihop8=2|j}JheFr8~3M>&k76pYvdm4a31)qTo_d5{C9K&~^#$7--D zkOM!69H3xqkLud5>;n6v`}mhzKqoNY*n6viV$=(im_dQ=}Flo>PZ($tg;+X zd}|+;G4p`mb?t@L*o`rmMj}87kO(Yr*2iGRfidouz)sDE7bg|z6aCsbuLri3#j=)k z*;mFnIhF%4s^Is$SPq$r{9cdlm-m`4D^t6l9{-%HuKL7@6?oPE$_-hWzLbmE8bz1p zMlr53`Y+!#?0Sj?4IQoaD`T~X&vl}9G7ZDfB2$*QB?E$6Y3n3r0nivM)DUvtTjP=w z{NDU4on5c)ZiQJdLH;_RLo%Sc)wqm|X(^j&K90qv7=C`;q9BYj4)Ps(%ZHY4E5|*bNMxEe2DU%Ehx*Bb>h$Z};8IbLoh5y=i|Z zKkFMn1psTw0Yldh(gUPFdu`5Iy!JY1oL*Y0l|q6#mXp9#OMrpID#5u>BBp_FAo5U+ zcHZCMu=@BSS>*PM;X(~faK1HhJE^*>ZCaJiOvd$N?_D-Aar0pgUHJKkL)`(jCZE7#T0Rnr7W z9c^=3q$+Q+1>0dsBx<=^|kb9sXi^lZJ(UJZccb9g=g77?R!*^qfec&LMVKK z!C68JR4+Gt*eW+(^xt_Y@O!Vc)9(1alXOIh)Y>8E6zee{LxC}S3A_F(Tb?3iY=s$H z&u~GrzXwl%Rd>|2!Ez(#UKdPC%yf``-*f$bCzJG&1)3Vvm>QOl%&m!ZwXq_zD5a6>MJ5QM~ByxN=q zy4-vQ^YMy{mv9&->%4#6U72cl`Si*SZ4S-|Y(|@}w08g%XYB}%yg1A>`NPnCWyIPc zDqscDd=X0N?4mVB?D91MSZQqxmo}=wMdKb@G<`si1AzUdI*)96K|ycU8FIn{N%KG! zx|@WPTh^SyLBtdh#~?SVTdZ}t1D7Ck=RgK^PVB5lSC8$bc+gLh_O#YVa+qsY9GRv?1`Y$uVq;gnDUjmCqb` zH>dq47idarY(M97x;rpDcEKj)(5eYw>tgsY(f$L7TkzaQ|NM!oR-_C(I@BA0v;=&k zEEZ)3bZ_7^*icOG14y1wt7Dl}jGyJ9UN0B)KCEZB(0lk5mZ%(Xdb3U5rX$hgJanSV zV70)de2V`t0q*@UPPhCJmb!kuJ2XF9W*D`kxPe;0yXc}mo*okY0{6Z0{Sd$Qiwy?Kp3)g{J1v5o9Pf z*$13^xN;>a+-(GHaf}Kcq0}?N+9+S105oH_^YuKTpz9h9IuY6v86Fe@e*`J>x3TXD zk$1Ku*ZmE}&SuwdjkDo9f*7k-t12BaZZc9Af-uk(vHoWC#k##mi=N%-5-| zu^F#0c!SEf$oSgd#ZB}L_Z$^HA$Z;}!8e45iiC}?2)(zyH3zrUb(ECR`kJ9P*M#@( zYIPSXIS$bzhFXqnNo~`yE3V9QzQ-U%5;cihBQY^5RLhHIKtcZ!#c6Px4Q_|Vx0Krb zTj7QQS`<8{4`1ZMr9+hST%`D<&u6DcPsTN!aQT`sTK)IL`&6<8s;@`t(b$N1a5!~q zTI~ig>Lh{VsyA9ZZX|OEN%l-GMU@;-!d)_zL?UnfCNs_YDdc!>Ml~V>Sa9!O;Ui%| zHWFcPTSXOsyISf3 zWZEbo9^InoKw~50`M$d@C_m)*nG})hH{13o>IR^B`iC@<|il_O`}SMK&>*^K03 z1f%z(?@jyJ9=oal;1P}2tULHQ`$ttYs^-wCmgrD!Z6%YM4hBa>p(XxkGffcmJz?te zSPC($ci(xUUZNu=n^;N%0LVQyqRZ-q>gueT#f(5-Rtrzcn6ZPgY;OIB-L>)Q82pZ> zWE+9NE*N=qt@Bwj1nXikhI9c@pm%HjUX{0+{B}6#-Mjrz=$VtmbS^hinL$}4IWG0q zzabjg8^!7-P@h$hL)>(mU;C~W=r^P;;xcFj88V37R+iS@o+Z4OEsaIlGyk@w2lg^v z?apHD@LhauVkvZ@ej^?k+}ZlEYHQ)@_M@Nk`#i42wO!=2x!6l=<8p_ym=UvRLyhi3 zrmMrRoPKViD`cH(gDuCpVU&}?4rffVQ|1|!kVFF4sq7S)5xKb76PPG1H>%KV()>`S z$%VU9{4^>We@hi#PFfvhuHJF}4s22M;f7*go4}y!Ldq2_+G}7_@;TenjZ1+gB8inE zCR~Cj3Fq>7{msGlg>Ke4A(hZ`=xXX7IuzEXv?6U#N+(oHpB4PPxLA{o2ak2`6vtAC zoHJC5RH(Dv5@sF|Uh!iIxhiH}FB@Md2X0FoEpOWjI-cyl@^o0Ad_813QS}OC@&pWY z=%Fpu5J`%Gq;>Ke55m9=Q`*)3?!%Lu-n_aH97mM6M#sd{%?9YRS4}$b-Y`lX?;%q%B6# zXopq?1@veeH)fzoTyLZs7@Tg(6S{zLBDnHRP)$f0S&cjOKF%|A!hwvbE;K`hgM%f{ z?a%g+>Iep)Y#GWpO0{Vjvb1Rp=6zr;&zIN>xJEC;-<8H|GwLK9$ZeWDQjAG}y?ARy$s#_}?TDd@Mz7m0ONM z(4*JMLMN*6FX0L?tBD$c$%xHVdxy?d5mhAx79PUg>y|_8zYYxUsJ=&AT9w8EY80W4 zPPszUpxwmxMm~`-7J?3j`_epwm;UL4vPk`5dMn6~1PM4S96D8F+4#3CQ_lWY06o+O zSfg)2th0MN9-eQB|8(E@)tI+dV=UO>NI6qEGQ1PyppGN1W_&*o<98M5Vk?4d(O^fe zzA8oVN?(mzlHIXO^Quv=PTihC&fPF4d3RDn1x};fEdG=YQTM z{}*U#(xq<1@&l$Q4p#`^iom@RBTHR*dUlkv{KoxxX${$uo+>{W$Vx*fQapT0h_1$~ zeJ=HG;8_bE_7PnQp2*2R+%%dL0`7nwl}yuIr8eR|Qq(*m+OvI_&NOq}RSf6fUf|8(raAOGpcXF-&Uq1r{2lUl_e}ZeW$Wc-OPy42?Jd|S?W{%@O6pDX#W)W0;v@~+U zF?anZX9h^o+>WCE1H5_6>2CC&%=cLMIH_n|pJhZOkoeBzHr@a05r8J3NEtMFa6pot zbM=2*AsnJwAwxM9f`Mh$KE3+W!I#^6A0JjX?nErD>4HRoqq+GygJ!YT_BI1L(G2Q8 z&&ppJDbzn-jDJf?|NZGtgS?DZzu~z5{6r@j`%OOo&(B*@ex!^`kAixqz!@mk_tcvf zI4#(Td*cwx{QG4hhNuug@Vqm0=Ta=EqhpjG5!!9X8QHdlPE@QiU5E#5?5@!N^TEKR zVkpen1$hI{A}G(`x5rSi&{Sakd5J>|fGn}Cj|8zoFwagv12h0){wB>+0qE7!0eS7z ztY=FkQ|DHxCZL1y|Hi!|k-gM2gV3`Rr51hx3fJGO-7gq6jcsJsd~nzq_?S-`Q54Ud zr1x)BJ-}Poj7w)>^GJ|*kpn2VSgNCWxj*-RD?+vyK1u+iPc2a zGqc-ZKN1U?_Y0O>Dp?5CQa!VSqa80o`nyjJ-GzKl^q$~-XR~iPuJJlpM1z9I{Kc=+ zgC+Z|-85QQm9gK(TH*nDk^%3~Y)FKM`5;fz>E@hw1z;ECcf3*?e{Fas@Hw0Ud2R7{ ze_n2;xdVphZtcehufQJFe;F0})_ zN(z{T-tS4i{ulxd4ItgVa11~uPVjFW_8rStQG&Xh8RODg0*J2E?!Djd^?DlgSi&gF5+ZbM6DQ zy;`4(RC5x*W#a#7@2!KX-rsj`K^lQYBOu)^h=8;xjVRqMAq^tZDWHUelpq*@bT=rC zq@Z+3qqHEM&%Jbi_ul6@GtV<;=A1KU&iwYDdsxf0KI;?j_jO;_>(cyqxKxhAg!QgB zg<$0bxwevhPQxG9GjP*O=59`xhYuaqveJY{-TfPoKXEOe!CD}I)*8Z!T^ z2TngYpGqZ=t6uTR8^BPFA>m9=pD5C=db-e)tPjo3j0dM-gv1$lUdc-1ut+w1h!tG@ zl&}is#^^%B33;%&S{&^Zr4lLg8r4TbZp8W(I9?RHUgU7Ee+K>*XgI8ZqKt+3ln&ZJ zTX@w)oj8-BWd62Kg;=F&w*XoJl8!@mV-O#exTm7ONb@#2=kVLxw#>#KZzk9l=U4or zN8m9fMd&<(94qJ*vQGd&1$T^KP=2$D89E<{!O{=8z#Tr$QUA09?Q)(flSp!hJ}QjN zHEeOZO3bwvf(VE@0?`HTgHaha;6AXDD{TqH7HUR!++E2m^*O$WTe4upd;~XC3hjFLreK?H!ne^M$Y8dhK4G=LKAM} zmKg~9h2OS+_3dZajh(0!8L7irYWKRPMP;*3@;ut;0|q8Sx#8^SF1HdwshBHMHT|j` zEr~uf7yW#U-0#0ol{i#3e69-;D>WlHCGP?4mGzWujs5=i!WYVK;1%P-t8mJ!-vt7m zP~o+^4Zc3QWtNc^9nrdws9R7}&sRcqV1$B(>&xG!l#b7rQc_@hY?mzIQ{Mx++SG(r z(KkwKJ5U9Nn}exlD~AC=0^P1LwR5xK*5GlBYE#Z9+w*7Wh*wx!_&dyH1FHAV{;^wQEf4Y+hV7k>)S*bg( zR&1RQaUXECGZQ%YOx0GzL*0ET*6k|syN{Irn$0lHd_mp`kG5R5XQiQWjyB`L(A@aL zAdKD&serfq7V^APC+`euT>99X& zJAPP^71&JoB$sJ4)RJ(&Gz8&}^Vbsh$dRvap625M zy$Vh6X_Br9N0^j;%yHkO2Msgy=VJ}KxPNr&n@B?g1H(7u1j+yS%H&TwB~ZfefB$TL z2b+aS&fj+YYo>3ksJkzzGT?4rn6{wSR?#oZY|Yhhk!8V2*#&!}g~=M)b<6K9r^VPU z^>$p!Mn;BAvX{;?NWc^~^f(t%BI7C=0bV>tC$f}+xg5-jFJc1GPl?6pCZ<&-PqWG1 z0}e%>OBt8EFy`bC7xSekmpQ1I!&fPrf^{;-$`Z3P@g01DcGAB<{!iYC34uJe3@?R` z!K?Li>$cj(>QZr!?ORzYR8({xL4sj32HO#~Q#tC>HnJjeWedsIXHl`VrB=ETzTQFOn}CEoq>QV>)CXLzS_oJ{1UZK&yC|X z^T=!y#KIKAlq`nwfRh4erUDTB*f;Bm`(jMDX#QDPUN@|0+ zJ-pl#Q-YtosyzG=j95%j{+Itt3>npBFElf7TAHTO9_my(j+k-FwE@knPND+OE; zsxLc>^eS%5%?tE8=s9EYV+o)odU`#d=l@TO7(70(nBj+iQ@!q93HAaJzq5SE`duZ8 z2$8|UMJ;{_v0}r%OI=A<%ho1qZWqV^3xDLIzI`~MGGyA~26F~FAYdN@VR7qHtU>(- z-zos+e`aqAP9f(>Wh7?t&go>Jylz)Gw?3FN&krsPH)s7Cm$I}X68KF;VApR1{119C zp=0qeo0A5apa#L?RRMTqgKEJdSt=<;z9(B=mqmy52MRX-xEF1;`DT@~p-Q;};6hMr zap;-E^XFcP9w6a!BX^=iL5-Nq9!dqnHqeIib$JqbeBRL-if6*cB#^b&6-)s#So8G2 zK$kh%-(*STO&>{!$;1z&I5(}-zHYC_dJ*Fxo)b2YO4@GSybpPigVy}M*AIxgVA zsYo7v)g&ZdN&_A^Bi;uedqB`L?y>_In_dl+6?piiI#yXxkH zrzIkABB?Spx+aNj<+y$>Bm$XSnR8=>5=YYqC+n_hI8i)2xVeD@l&G96ap|hM;p014 zSCxbXP|gXQiG(&pXU7bq?Ew;?2Ovt`hSNRyKFz=hsAYyHOLaIb9P2vDrn6yWg;Xq8 z6aOz*_*uRGlGF%t+9Es(k!m_-(jqLoI1+hYkyD4HaB3SVykK+oUj>XzkwM*~+YXZl zcaZBBVOAU~>n9p#2lMVCz9-QDj#v5Sd%S+3N)py?u-eJW8B_uZYir&QF>_R)F9Zf2 zTu&A)Sg$cZS{s#qv=5T~!trJU?n7~?kxme(u;ZQUH<^e9u&hEdznlEPoEuTt9^9qg zo-!_>wFLZLOAB-5MW`1_Oy2h1L#BC*M@u+U|BWsx@cLEHsF+_hM=V9eG z<}g)DXM$B-f>&Rv2tBZZu*DDY46`g2x`ZzWl5HtNdpW>IGXw{Izg>5;@tNQ!)7NCKnz z8wL(Qo009fI$L$N&vD#`_aZpdRXZa zO4G%A!4miK(FY{YPq+aO5f2Z-jtm7?&1gK@Z>kW4a-TMgAFt@{EWJ;Ks?+6U!pCLJ z#Wb^we3bj3JDt7xt-_JKbeng%g2R$XD0_%u*_XyeKr0aqUV#o!7#CrH6ld0^*qLa2 z=f&f#Z$pPuAH4J-4$TAW{vj2por(vpg9MR)01(zhF)RChL#` zUZCYG16ShhezQxEGqh|=S@uvA4O5&H`krQe&Az9kBvg%g+LPX(M#q9&1H(JTf_Zz@ z0IKwOMi1USTpiXjT!OGm-rJ?MF{Whe<{^>GSl?oK}o=mpdZC#%G17H zx+}v4bTCwk_uPH?z!2_!H=gfoZgNBDbw~o%|0P_yrQg2pB^u*LL)N1MGMh}M&9?%M z(B^^O(z&}j-PIB*2}?$ku{w9Tv5%+O6578m&DLsXDsuazRXm6vvVx0M=0lVffpgvc zZu-uotFEul8WFo$s^?vA1RJdS9=#e_>~>!)sQTuTjqk{Sq6)js1qIa@&;_@nrUJl45E*yHLJ=w zG@o@Mmr&;+$9iDGU0f|n%sGCUK21aFD;&iWSI_W$2Wo za-5jV2Gr>T{o|TL=L^qWj@(R$&+Eh1wTJFFRlq72;Nv|0Ce52CCt7*8Z9he+pzaI* z3=l@Obt=*+3Bxf8yRMVOa)+877Hark-=e$Mv#Op&!v%=kBk!Ek??up+U7^djM9jOb zTua#eSjPa@){B>ds}pY1;S!F-RyR?jh_j6wVm0!#Gevcjg?h?RAE+=q?2eXm$IbanIv`>%%sqUP;rWA%sr7F|1U zj6xSLl0_TV-H~4zv>BShB}<|dT|Z-8X38WB32s*id#~T*d2xxX}WPlClLc z%$8w&zR(y@M)sw@%*xA+d!lMHf}+U+Y&_#j2*1oCocsRzhX|RUpj+= za|nuHGqei$VXdVi#4F49O4wQD8z?~U(~2O7ArVg2%L8e#dngz4UKMR#O1|Ni4=6)} z0;6H1Wg(Xi5iy2C$>DTgIib(UG2E7zT%dh?7Yb6uxx1oHhj=mpV@0(JeNq zfV`iEQ1ei3{GbX%qa!U^!>|*)WS*Hm7|Km!5%qt9%aHeY>nqIY={;5ps+hg@%8$HG zKEDVjZ-{|*lF2VP}Lu##zedxzL~b$DY|vH_Pm z5x^KaM*-Ea%?>l#-=itwl5w#rV@=YlDlFQFlT9@-AZCeBTb<0+)L(?uO&kPbI;_$c z)w*fAw2gbU3PbTFkQVpRF_4lY&6RwnhQ79JibMOAg3jpn~m>?!@`-k?h%ld*(5;*9WqTBGbQo*aD z*@VVPFxl19KJF>&q#HDZtpGKrJXuseIt0b{gqWF{(A@d#3h90EsYl=0UvvC+2V4NK z8L?6WDu--7?)8}~qI;!SEE6gcw}|HK$#{)epWygBcuLDZ%&${!XPyJR+>BExKKd=x zx$IPKX>6AApq(kW)5d}}n)}1L|MfFC@OKMronnGk84N5S?~&&kikrg7ER0J$S2Wk| zBbOqr_zU7E9S2(ino`YfTdPG?3J~PjJ%XPDo#!SbhH((*q>~f~%qOVYx8=I3oxhbn z-Vy_&&NI5hi~pQ$KjCbvzKLwG!cySFyE3jnT%gZB%=1A08gP<4ZH9Z41ak2x2W76? z9q0)iJE6M{G81e(f<5aXQLsQDg7%5ZE+j+C8<0+&XC&8D`V35rlk zD=^{5#;mWe?LbOnFq0iWTk2iS@H;Q$_dCO3H>eJ-#9XCNa|Mi#QJi>ig-KviurN3m zLfUSax8j;%{#xYB;PI5db@kEO!yFiFtLHVRneaFFS2Mqrj>AIrX?`nYJDhO=n+B1n z0rMWEwU@hebbIYlyJWmYpVjh{8Y&2^#1=c_#)!HteI(5~K)J|}SFoA>n761__ekX$ zI|i||@NSrgz;E@}9}}sj&V=^m$kCwEV`l;PRH4kW;@U?{EfyVzEaj~)tvK>){z;5F z4^2pI#!7?q*ozNkiO{RlP2YWRn8H!vwNK$gxxA|Q;=rKo7BJ}rzo4Q~8XQNvP~AlJ zaa$c$C9u4WA;!cCGx~%}S5}BKOqxfZk#@f^!_u8?k4$;fsDg^ydfl`>j5SablP46O z9C7eqnIKh8nuoQD#Y`jb&~3hKq8CrFhck@*eArfp*T3dJa_W;=5^d#e3nP>%P^pf} zAgM(E6(R+XuvnlUh)bDaP1>sR1sE@X^$=`A{TzgfnZYfmhRM_dRIt=`7wP_T`w4|_ zzzu-=NJ2YvlN({7XV{zZGOLu@3F(ezwb1X&t95GWr%*q%bCxVXt^m5@VBmdmAgS7R z2~DRau&XcGMi2*I)0r39UlzEwU4MZZ`!6;9mKi+WV`gUQyv88LV@-_{Zbc;3jG=?b zCrGGx)OnFKZlMX$1=bRW;5cR_dbn!)TM3N)J{SsACygfHkmjyjmI}+1R>teLf zgUI#HI98?TaDw5hSkI%b=v%Ms5iy@k%_s+8Nuy#<*m)v6fJ9X4^rSyU!YAN%`8bV1 z_#U)A=dq97qf;gvvB-CoFW5+r+)T}MwBWz>=oZP2mAEX{>Q(cQTna(-jh8_fk>weA z`#nNYWh^LusKu38it)n^lz#kx%q&*DAQwz$(HOT^fIN8iu&Iohp}!SFFV#O;zf4OU zRQ)zOmQY7^b+aM86K)QJ+@FmCtgF2b1KUZZBcxk(V9N__K~a`lK^;yNbPhPKO-YC# zQXvNe?UaYv5?}M@os6)0!r0^`Jy(aHgXnIsPCU%D8ff*5WQBq_-?q|h_#1_D2ni4+ zloIa&+=_VaTJ$c#uRC!puNLhfDY}J0^z#CE*gG%M@V}>6LVJu)Jn$^^@GXFaVbIF< zx4eREDN#1?#UF!WXdA$%;JRAdAk#R7G=tvdm zhI+ z@G+1$pnGG{d*uB3*m3$y;d83U8=L3QUuf*V+Hno~Gpv2;&_*{(ABz!1m z$SiJ`kMKI&tdl(V=Diy?R`wmI5d-rIIk(e)x&y z9SR85);|#JnBRU}r^acP_334Sgm3#RLk(cx^BSvie4lBXnp1As z`$*~KBUhUE-BG7PlA}@;W(S|=A|2$IMDzao>QJF9DIP&##U`!Hcq)uv*7l}ivw_yw z7d9{e0gzFTO$l5DI%#Ik^g3*GIJ&_4ifRyu)e_LY*Oj60?mjH=19)DEhF)7fTkh#a z`5+uYp0i=dE*wn0`t!$^?pQkMo9|O@p3=*Bc0wHuQlbzxOeYn44noX#ao5kI&>aaQ zu)?CZcgsPM!?FI7|E$8KS$1r^)Kqi`*6`@1b~@96 zwz@v0>bXNfheuf+4I?lGU?FD$1RI1pYH58ik3ngNVpr?aGIaBoBCc<_^ed$R_r@L1 zp-FHpW3!=wMZfaE#t(ugu{?%HXb5FtHi`sc>LdN^yUrpHtWrMzy zs%RR$i4S#&*X+kx8Uq@JuUK^_=pF6vC(N~V>-gf$#xdOHsB_c+!{_cvHngaqSL5TR zcv!+DvqIPs%rBj^Yq6w}J;*isXFYFy1@A z?1DiEb1@j<&uL$Q@wq9@*bVU6fx+o%Z9~)B3+}>p{m$|Q=NM=u3`cgc}MI}o;K!G^}Ht_t6(1Vzy^@(mFRwn&XuK}WDY<~7CghAFym52V1YFo>4N8f-BS8@><(l;9e@_5HH(r_+G*jOpl*Hhx=3pq6fYN8CM@qYYKc(a+rZg2L+aHYM~ zDd~p&czG63Adx$aQz^-yrZTI(!CizFv27EYQXljnEH(zXlG%RwkB@ix>y-deb3pLH zcuo*$rClf3u*+0jOjzoj7$eOAyB%rRH7pChVp3&9^rLc((r3sQM-u%+U%S!O(E(jrMCWhQSz!7(B1`(fj81iyd1vAns^G z-JYY5Z}GFHZFF3-cn(3JEZxv+a9cfkuxJOSjk8PLhVNvDG7ReqkA49?4sAZziB`fp zAK+`o$d%%+B^B*^$!zzCD*VA0)U_cOWc~^N!MW>qom|2n;S~AuUDt($ zQ0~tuqLrelTK?y5MH09Squ1Jdc7xeZYTHc*Y_3nhMuZL|e{Ech>nUNJhuhsK94wZ- zdJOc2pO&2ZU(>{bun1pWg-%`J3B_C+Nm7| zPKgjOBrXJEQ~6Xk9v-fj^jNLwI3lHnTEC{njkU2jjsv90e=~qC@HWs%n^4{BUOy|j zql?SZ$^sInH1q*>1Hpj>sqjErAg5m9q+jxLr;hndJEjr@>qQ5@1|6Dxt6ZFGbIiE+ zF84+Pa>xsMc~Rz_zqK%+0zue77Jjsjv#B5*mfpFnfB{te?7g@(rpFsE&W$j0Z>nI=b#$JcGO zl-l~iJGOh}!9?Yo_Ti?tA>YU9J?ue}@CWy?&d1ZeXA+J&KLUxPV^#=oF^M4~m4zUL z1qs%zKPq_~MZGc&c&uxr(;LH zUV)v}4ctcccj7o3anwr82rE1xe17jqwp)PC+DK6zH0l^Yp=E9Yp%vNyNZ6TBh^U9I zCCT*M{|TaPCsahpPUkkaSyrQ3)uZ)0-IOI816W)-^RVmU0UD)R3{I1UeH*_U3=)cZ z`X$jQ+R+G2r!cvM=G!Chq6B{Db6e5zIYUse=?p+g1pK)(ZK_EiMY&riC%8-jyvrgL z5&yKJjI|xUx=sojh3-PIA;IHw( z-Wi8PDQGpz$8If-NrCK#)oRH&v2rnwpq-8;ItG`x)0n_h3TPDIgcGTsg+~17Hj6Sv z4v)zbTykw(!s{nZq+Vpm?Ox9A#W@UTgD`J|IL9Xf>vH;9w({ z;*+}m@rE1a@+dte;jdr|(mkTL-R+x=II3r!9zC=dMk97#Qx&R9<&oVo)kWlp)w5d{b$0 zfV-i)Hr=3vY-+TBT@d}uxhY7LdUAMsv73_40k@f^`7)}wekC<5-~Jb~8=bmZ!widi+07mt)|;c_O{Lu|^{?wLZ`g%?h^ zEtVC!0=x<@ujvM3u~a+J@1eI@)CbwdKYdq9gEqN~2)`eTH;P1N1h)NvfY#Tl&AZ`V zWUaV!o+i8h+u#l{R%ZVFBCYz+8p<7r&GQJ(oj8#CevHo4hN02yp{bu7YzamcnzcPZ zkszVC_aj^BThi5m3{SW73}V@&6k(=RCCvgog~$e(z|fs#fv&Fj6iqv`e8{oAAmuzM zV|gpZXI%G)`R=fkT)N=qpRst@OXR3-$#p*a-DQ-x9h41?gO^c_535BzJltY7rjGSL z)-&qR=+;7->Rkc~wZ=2NxBw2c%#^?LqLT!|?4_Nb>?Pu^VC=G;8?mx_3F$ zyTjNgm;D2$Cdb=CPp6W=yrkW}N~O*;rL?h8#uiF0Ij6*1A%bzF1XddKy^m9_ze2r& zic5%Yi#yo?Hpxug@%(VmpI57C-%^C4xAYRup(|Wb*4qLKlFd?8Q;^)UTNrX~uA6_S zIBeERhn!G7xLQDGxHzl0Jvxh;#~Xp3r|qat?;tL#vk;AjQuw4L*wsSSRAIF;bxQY-)_YcA1N6V$oLGS{V@<@szJ>Zf z%yB4l0EN_|J3(CPD<*MXVB`*5Gu)UXl$K0#wxFiQUaT0Wz!Z>Cw(KVOgum@zLws;@nbOB9-3(CfM zn zI&8{!h|iU z|CZxWY}!Kh@_!1*q5Z!DIs6GUzr6zMEw&T+*+O(b#R)telC;=Q1XltM4(9OPGmz^1 z+!jO-f4EiOr$6*x4=9Q<2t6+VI5>`zou}is`ZrYC)hA<7ZSj@&$cHcLSVVZG?DipK zoN)7uX$vES21)UM;cTLQ36QY`xNV#ikK^o+&}Y}1s2Fy8F%+$pQayBOIIHNFR{}+M zDBS4#O}KAvheuDcsu(*N28GkM))2->_mW#(ZE6sAT_V^u3R4La9c&xzKZi-zu9a8c zrK={Y>1R1xs+K(2&!1yEB}2*A8<}hsO1CiDVy~YwX%?N^F$1e4Yn$Ibykj6SxBPRe z{uD!QSbrwVup!wa4Ye&-VAF~taSMNp8C8}qIf*He(tOccp;3=-tljE%H@2(VpOo+{ z61Xtgf8*OvD5n^jvgv(!rAe!lOmRsTSOy}$KL3z`rT>P}a8B>LzBS>vV&DUikdeIe zqteVJ8Myd{(68eU`=GjjR^0P$^pin#95DGm{!0I<#X^-0vI&8hpXjS2|E4uKP|=A# z5=E&-We^nk%!_O5ey>V3>LM7NfHu){x0IfM0gNh1T9e$};{WD7{Q0n7!eIGu%`+)g zG5B|w{8@3EgxcuzxW9G38y=FNy!StOCwi{Rt9)Vosj1YX!&DDdM+R(wvhyVzZgnPO_m0ojsE< zWmY-B1c`I7ni3ws*!+`mN7`$C?oC{6?al*tC98z@XOTg;_rr|r!()RSYlH3UuqP2d zAY^a3LV5>wPVvl2CkD%DZvV;OGh)BsPB8|QgiO;yjB0uuJ~vW{%<3<^lDIO~&Ud;y zqVCqKU(RMR3j_e3H&%beGYD83DOd=64^lm||2TF%x2n~yW^;K&VF67+yQ7Tf&%l5c zIWSmw-&Yfw@Y^zCJrbp~d-FzUr?GIay1Tf(wm8@h`pfwLU*m;Ph-(0mBAfgP+GOTC$%D*8lIjgXL^5Sbb%1xfSeWN~eFl_XX_7NL+-f_eyG6-tPlsU?}q@ zcQfCll=Kd;l5sQ}@^Ag+bxZ?8QDdXDhN3(}6VUi=03#G|D$J^h=Ktf1 z6b^vOCdJui<_zcpj?$ic#RR5tuwik+#4Psg^n-Xv>;Fl)d*_3n5{I*)o5)P+>^E}J zzheSFS^0oB)R}hb%C1w)hAeaxmY?Z=^TFlU5&@`1qL{X~PA8B=8i7meAh?_Spho@A z5JMMv1$9Z>z@Lo#qjlHaKZJ|o|85Y&|4t5t)JUOy#=~VCq$!$!wP`udfByG5Ol{0< z`j{2Mugwi9@4i zhT`%D$mNq9&IF--t*_DcfAE~r$|C^4z;s&$(8SHxC%OIA&kL zn}li<8EF?9eP99;Cw{-(+Nkr+DW_3=d28fx)QtG<)@2@8>BURKrJGX zNXU4k1a=3oiWZ>m1RR5vlLL-I3~=8%q5LN3icC{r+d}KpmZon(4Y$LdVl9hI)w-!e zK{fNQrGjW6hK7tyjv-)9Zw{vd^*l+TyEq^Yz>CJC5X*!+;yWsg9enFTryMfHfDE`V zbgFj&gpo_r_v|q3nvI$ajl{{=b{zmURKab|0c39M!5uIHLBTAo4| za(1FxOZY5I4eY-;x@BqJTeqKw{ov_28m(~+-vQ%n6wL=NNEclyd-Tda8=1^)AsD$2 zS%qtD53RfY1WAjeHdU<0T{7{P)UuRsL%T{kwQ#XM$W`p%Jc>qUD9Q+EPPmXBSoeOo z-$M#Q3u#VMO6WoQDdd4?yZaH`?6H8y>w?%4C+vd<3E04r6%RxBY`QXSG9H8ZuKp{z zxYCaS4h=}4Aenvb95rPRG}dCIhtGDHD{*(#y&G)n%)%)-7M(8q4wDfVd}hqmKBr#J zfV(JUXf%^W-JdNUB@3|bPQx&&_ykCH+odqYH`#>Q5QE>@VW>3@QgO>Ke?z|F*RwOaPxTXO2<6KDMeHLNJS2w_d@=LFVeA~UI z1#2(*-O$w-X}%BOr7a+i;^TMa&yS+6u^8(YFY#^?UO@EKzn!`F2AQm8u4#vXW)fILrZKjAiNcigH06UIU>Oq)G>P% z&AZB22`=BY;kIM{?-YY>ayQEN@61m5V{ME$CQY&YRPK7w$q6vw47E&3&*y+3g$l0W zUl@3~)`s3vIJszU&)@a^iGA41j5VC6Le;?}Aya)QTJsYKp$RA%RdeM>Q)ZgrJVg}A zp#>OyxX}ULMpif}bfz8_+cYnAzA$TZS=o$qTWWpE_%%!A;s@`oxpgV3SFh2CH-xAb z@4aJL4TwReXh4v&SRW@nx=*E*0~a?E*8&2ajmys&2i@9-L3<@$`&!9@wxuxQG&X#4 z7wO1l!x>9RA_7j9 zvNktdp2rX#PXuhIJPb0BOqEiTrXD(OWu%}FQNRUgyd6h3SO&aBfxWpa&E$czWV@Kn zSGIuL^RL{$#XxOQ&V&X;Z30iWy&P+YcV3F#LM{7u{{9R}K ztHCaFt%5+a*&vs`jgoOMuRDAy8P;}arWqadKCBHdkvTF@ppfs;bIYQ~{>&0w4 z{Cp9(t!jm-4aZN#aixJLZ0v?hcA{gb|1*<}su{^tL5d{gc6NouV-Q^kH<7`w;;tqz zjkDPhq*AOt)Xdem)evk2j>G%P2zq!93`de(Uxb?`9YjF!W@5<=8p7_6*13lQgv z2vP;`f$uD{;2XcA7yfvD_Hr;^S5_zhirHd;carTSh26zWK=t^i-B50j!$Pr5$lV(r z+b=&4@l^=av%5iG4gm!AWC1@m_&o8fs%ag7ZiaO1=*pv~ zUn^{l4|hjG!A5n7{qF0q+A2JXeifu$zRjP#*bV|JMg=dx(ij7VuspuT5HeJr>XLDu zF__Qg0wav-6gi)3Mm2a&+mK6^4)(28P<~*#QaV_$y7|a`wpo?`u@mp&Q`DK3D&f? ziZZ9v*35n3mwzP~I(9bo+D*U7OgIBoSDQm4JBy4IAj-CmZO?9l=B&(p`^N}Ln9$Di zK#}9a^025C@K^>)yC)|2UG~vc6=dIHjTeE8}+ZlTwT`P96yW#6DMxtPT3?vL8 zD<@NK-?}~HBkX9}> zmuGzdOPg`Dl<;qv9p^ip@_MJzGv$Dzo(PV>;cR867l zI2w`3%hrU3f|@rRaM6mpFbeS+ZZ1Ox_vju)H6A zt#ULG5`gpUR~W72$43DRC!)VrL!x($2n3*an=3OEi<} zl9e+TrQ~@$9viD?DKigabJ8V6R96U%{3KG25@Rj9zJ(IjNxQ~#`-~s;k8IM&k30E$ zR9QAahc!y9teSSh4ldDL!AeKTTm|art59~)x9iTb(NxnD%7}~T}2`?BTchOv?z>8nU0`O#;wi@$;+nh3&=gqbIZ*w-Cys*5>neSE~3Nzg=^?joa0X zmVhur5GBzXgxt+XSHC+iXQ)%3FBFDkyoq`6PgL+VJz)haajl4~)-`oxXwWdFC#!pW z1(hD4GFLtfh1GP$+sX%`ohe*2Aq~*Pnj?GvN`_84f4u$e?6;ZUB&#jCsE8rYNP0{3 z1choFquPhk+)uNR4Hwa*SDA3*#G5Q`IsRH&=Q119A^vg(PXBQMk$z;eaiO>d=_oRIXqR$%_Qe5ig4PD2df-H;&X`9`s9EY0B!0J4P}Y6ky5m;{sryZgA7SO-Nvh@(U_Bp53zHm zGnIUX+_0%WprYd-w{3ZM1;|+JeBt)^4)kfdrOxSIR;L6pog`$jA{qa@+Wt(Fk4ly) zcTynbd7tETB3R-vMvG;<$v9r97@pV}ZkvdsvbJKyv)yB9yea1r5_QFEIb^i8Xa<8= zJH(HQoLk4yj++BE6JJM5!KXDIX)5Cb7b?|gpJZPi8OJIRyH8y#@`vO2=13YY_JRE- z-e%xLs0QF?Jf4vAdOeJd#XwTMFh;B#Nu>=92xbY&j|8jt=|5tjZ`^g7F5_v!W?7|> zgo$Os>{6JSfy~w>5VdSqti@<`PzlvB$R*611bi3@{9rAQdb)+v$;_MG@TfyfyTO+^ z5+&E5CTq{0^U_VKmSA`4VZr)+uRXgF`o?E_Flh6^T;FOHM<{~}ZX}6M9ZmoU{_3H{ zdrU_@;esE8>%s7_{SGmV8`i`*Gc++Tx?3e&dT?cHKkfIA+{w-P3Lv?_fcX8yaN_jd zDJcn|LQDoktcKfP7ay>@3yYl7x`#I00MZ*c|Iub0%@+00J zyr%U%v^Z1C^=8XkPZo))JM%NA9NaEu9I79HI_P>vn!>G0U2;WUEg;0L$H^ z*ReO_%dv6{bp+YCAEu0)+-$wc3t0V?RCVzMU1`sH5XpAKg*+}{EI8YAo2d?@gFIDOnmXS zbY;?^1m_1>BR9dk+_d;~HP+W}KX5A(W!Z<86h_<4(i3+gb6XlH{7(T1UjM#6u570rX3C zMl1XJrGFz6pJvSgS{BC|rQv-O4khXBN-nCdknI6dDz*UjZspQTta5u4nQ!$SpbyNi zzNk6DNrNMKePUn7y_4B^8J|Y1e{-5m$X548+@jxmno^GkojSTaTDh@{zqE;yTv{Xx zrq*%}f3}=IJl;S5c>Lqt^V-bJ52q>?x#8kJ8;g#pKwa^h)YhCRW%klY+fefr|D+he zPQO(-;jnR0HSJ=2N{e?(+I+HA)Q)CtX{e3bOm~lRLXyvR=liLTKF=Se`|R0NS@ji4 zEGXkB-%r*|kj$cx+7n}5up2MWoF?D<+0yVMTxy}_dd}Abs-7Op1s2KKy>Z*3mHqdb z{nC~JM8f=g?26IfmF*`F3VbnmB%cq>&8Z$MdX8PxYG;dI+@4lLF!(a_G3&6HSYpo& zY5Ua~CnPYN#vR@8?Mz^{)iyfpJ^MxM(p6etI^yt9Qjvp@rToP^w5;v>Qm9ZV=(t%} zJ-6DebXaP~{XLx**X6~j^ZiYOW@QJX_oO~y_bjVi7L-B`GU;wwLX(yllVDo1ea}`i zZqc3q-+e94&A!sM(eD)8`{FAuPj7 z#VBCrQN4~%Zt$~)HuLq(wCT#~gKl2}^}R0piAS#I<6awwy64F?pSL%$mum>S%ia

F#X@6TypEU)XD2P~in)nem4S zJxjx*JO5s?Lej7Uhv#Be{bca;!vTV$lyTN{@3-lTiV8a#%N#;A%Q6nx#3Wic%DUr8 z+w0hRA}(dGd~|j95*f?*-{UGDW=SeV@z-h5(4{(lD%04kUHw>R7GKZR-Ho^Pqk0j% zFkHLLmBcvn)SJ}I6p4S)kbLCZ~LamjUK#JC~H&&N`ph_ajIFnIZR;BY4Vo^$g{^pMcA64Vh zK5k7E0md@d?|X}9_l?z1&7z2s)T$ZL#au|jxAqsUu)9R6dW7T0nztG?a+RXAI=jod z6vx(rGd04LggKSA%wGB@UT#Gj+>xZ8Dfx8Pzb|?FlI*@!Ryu(yf=|)?u%jcJD!ZK=W6N|6wzbm)heprRy*IqP9TstxNCe*KrZ1|7MJhM zUBy0m4godCkB1vB_g*LjKwMXzhW z(lihFRKne+*K_^Vmt*uS0kdCOil~Rzh0ElyxyOi0wd5RR!4WMW!v48fjWD3%f$ffK zRz-^AVA&2wJc`oO#o%~57hVDd6_g10FzyZ7*7A$F-d^da%Irn29GA+J*)xYfIbLUd z<+x|On5;YI@3)-!{`xOf(NgcN5Bq0BlIfN|yt=Hi9+>X^%;<4bta_e4&)(vGOjrIj z$h=II^AV>Z@8Oz8S-2IAcZ#_n{EQx!TxDxc-szglewJ(lp(M;n4-GRP%XWQct$#7H zP!As64grZUZE(ei?_-CBcR zoAs{s4VLQkyxq9nF;{yMuaxxlh0h(Fd{*ox2Q9JKLl_3 z;O5iaH$K9NMF0wobTFM2<I2V z=#)sx47Loorwxw$DHt?x+R`z9<-mI5Ve}C1Z{Xl>k7Z-QxjZA25H}ARL z{#u!wK6j^Us^7;s&&Y zrf;PAo$&^xbiHF#^YvaGp1^Fd`cQw@M30woS?r&`y+lG$8_F&Xl7VQQzC&`uYYjH^ ztN`-(M8f~P8$yQ{bpSQ)^-eLijRv`k)99Oa9=9K4KweE?Pkn+vgte;-W*9bY=U(mSfIjOAZ1 z_4jYQcA-=0Ki>*IUy>@kQsUE;?myp!j!#O;gwlV3`R{L!hSl>PUYS0>k^lEug@vG? zqvOMq+9vwX*F>Ib>QW{z`<&f}|Mk*K2t*PFod@>6zWXnBI=b_Ic|IY19<{}2EM@tF4;M=rk07e>pG$@AuC|m(?rM?Szdi&d86t_2hT8C-i$m&lp?{;si1F&b zKEyRHIzB-$;ab^$eO5#GMuiQ-|2j|p>wx`zS^n1n`}^eluQTlL3keVRgxnTe&LY6O SC;0;Wr*Qkut-_lof&UK{jXj?L literal 0 HcmV?d00001 diff --git a/docs/images/BYO_DHCP.uml b/docs/images/BYO_DHCP.uml new file mode 100644 index 00000000..1ee008d8 --- /dev/null +++ b/docs/images/BYO_DHCP.uml @@ -0,0 +1,36 @@ +title Bring your own DHCP service + +participant Machine +participant DHCP +participant Smee + +rbox over Machine,DHCP: 192.168.5.5 represents the IP from which the Smee service is available + +group #2f2e7b In firmware iPXE #white +autonumber 1 +Machine->DHCP: DHCP discover + +DHCP->Machine: DHCP OFFER\nnext server: 192.168.2.5.5\nboot file: ipxe.efi + +Machine->DHCP: DHCP REQUEST + +DHCP->Machine: DHCP ACK\nnext server: 192.168.5.5\nboot file: ipxe.efi + +Machine->Smee: Download and boot **ipxe.efi** (TFTP or HTTP) +end + +group #2f2e7b In Tinkerbell iPXE #white +Machine->DHCP: DHCP DISCOVER + +DHCP->Machine: DHCP OFFER\nnext server: 192.168.5.5\nboot file: http://192.168.5.5/auto.ipxe + +Machine->DHCP: DHCP REQUEST + +DHCP->Machine: DHCP ACK\nnext server: 192.168.5.5\nboot file: http://192.168.5.5/auto.ipxe + +Machine->Smee: Download and execute **auto.ipxe** iPXE script (HTTP) + +destroysilent Machine +destroysilent DHCP +destroysilent Smee +end From 4eaa8bef44d3be841ebd43761683483328e0abd6 Mon Sep 17 00:00:00 2001 From: Jacob Weinstock Date: Mon, 15 Jan 2024 11:01:19 -0700 Subject: [PATCH 17/17] Fix linting issues Signed-off-by: Jacob Weinstock --- docs/DHCP.md | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/docs/DHCP.md b/docs/DHCP.md index adffdedc..6db3a254 100644 --- a/docs/DHCP.md +++ b/docs/DHCP.md @@ -18,11 +18,11 @@ As a prerequisite, your existing DHCP must serve [host/address/static reservatio Configure your existing DHCP service to provide the location of the iPXE binary and script. This is a two-step interaction between machines and the DHCP service and enables the network boot process to start. -- __Step 1__: The machine broadcasts a request to network boot. Your existing DHCP service then provides the machine with all IPAM info as well as the location of the Tinkerbell iPXE binary (`ipxe.efi`). The machine configures its network interface with the IPAM info then downloads the Tinkerbell iPXE binary from the location provided by the DHCP service and runs it. +- **Step 1**: The machine broadcasts a request to network boot. Your existing DHCP service then provides the machine with all IPAM info as well as the location of the Tinkerbell iPXE binary (`ipxe.efi`). The machine configures its network interface with the IPAM info then downloads the Tinkerbell iPXE binary from the location provided by the DHCP service and runs it. -- __Step 2__: Now with the Tinkerbell iPXE binary loaded and running, iPXE again broadcasts a request to network boot. The DHCP service again provides all IPAM info as well as the location of the Tinkerbell iPXE script (`auto.ipxe`). iPXE configures its network interface using the IPAM info and then downloads the Tinkerbell iPXE script from the location provided by the DHCP service and runs it. +- **Step 2**: Now with the Tinkerbell iPXE binary loaded and running, iPXE again broadcasts a request to network boot. The DHCP service again provides all IPAM info as well as the location of the Tinkerbell iPXE script (`auto.ipxe`). iPXE configures its network interface using the IPAM info and then downloads the Tinkerbell iPXE script from the location provided by the DHCP service and runs it. ->Note The `auto.ipxe` is an [iPXE script](https://ipxe.org/scripting) that tells iPXE from where to download the [HookOS](https://github.com/tinkerbell/hook) kernel and initrd so that they can be loaded into memory. +> Note The `auto.ipxe` is an [iPXE script](https://ipxe.org/scripting) that tells iPXE from where to download the [HookOS](https://github.com/tinkerbell/hook) kernel and initrd so that they can be loaded into memory. The following diagram illustrates the process described above. Note that the diagram only describes the network booting parts of the DHCP interaction, not the exchange of IPAM info. @@ -56,25 +56,25 @@ dhcp-boot=tag:tinkerbell,http://192.168.2.112/auto.ipxe ```json { - "Dhcp4": { - "client-classes": [ - { - "name": "tinkerbell", - "test": "substring(option[77].hex,0,10) == 'Tinkerbell'", - "boot-file-name": "http://192.168.2.112/auto.ipxe" - }, - { - "name": "default", - "test": "not(substring(option[77].hex,0,10) == 'Tinkerbell')", - "boot-file-name": "ipxe.efi" - } - ], - "subnet4": [ - { - "next-server": "192.168.2.112" - } - ] - } + "Dhcp4": { + "client-classes": [ + { + "name": "tinkerbell", + "test": "substring(option[77].hex,0,10) == 'Tinkerbell'", + "boot-file-name": "http://192.168.2.112/auto.ipxe" + }, + { + "name": "default", + "test": "not(substring(option[77].hex,0,10) == 'Tinkerbell')", + "boot-file-name": "ipxe.efi" + } + ], + "subnet4": [ + { + "next-server": "192.168.2.112" + } + ] + } } ```