From 27765ca9f5be67139bcdbdfac84342d447fd7237 Mon Sep 17 00:00:00 2001 From: Michael de Hoog Date: Thu, 26 Sep 2024 16:13:33 -1000 Subject: [PATCH] Add DA server --- go.mod | 6 ++-- go.sum | 7 ++++ op-da/cmd/main.go | 32 ++++++++++++++++++ op-da/da/config.go | 22 +++++++++++++ op-da/da/filestore.go | 30 +++++++++++++++++ op-da/da/s3store.go | 57 ++++++++++++++++++++++++++++++++ op-da/da/service.go | 77 +++++++++++++++++++++++++++++++++++++++++++ op-da/flags/flags.go | 39 ++++++++++++++++++++++ 8 files changed, 268 insertions(+), 2 deletions(-) create mode 100644 op-da/cmd/main.go create mode 100644 op-da/da/config.go create mode 100644 op-da/da/filestore.go create mode 100644 op-da/da/s3store.go create mode 100644 op-da/da/service.go create mode 100644 op-da/flags/flags.go diff --git a/go.mod b/go.mod index ecc179d..5983633 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,14 @@ module github.com/mdehoog/op-nitro go 1.22.1 require ( + github.com/aws/aws-sdk-go v1.55.5 github.com/ethereum-optimism/optimism v1.9.1 github.com/ethereum/go-ethereum v1.14.8 + github.com/hashicorp/go-multierror v1.1.1 github.com/hf/nitrite v0.0.0-20211104000856-f9e0dcc73703 github.com/hf/nsm v0.0.0-20220930140112-cd181bd646b9 github.com/mdlayher/vsock v1.2.1 + github.com/prometheus/client_golang v1.20.4 github.com/urfave/cli/v2 v2.27.4 ) @@ -55,13 +58,13 @@ require ( github.com/gorilla/websocket v1.5.3 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-bexpr v0.1.11 // indirect - github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect github.com/holiman/bloomfilter/v2 v2.0.3 // indirect github.com/holiman/uint256 v1.3.1 // indirect github.com/huin/goupnp v1.3.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/compress v1.17.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect @@ -76,7 +79,6 @@ require ( github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.20.4 // indirect github.com/prometheus/client_model v0.6.1 // indirect github.com/prometheus/common v0.55.0 // indirect github.com/prometheus/procfs v0.15.1 // indirect diff --git a/go.sum b/go.sum index d09fedb..98d1ccb 100644 --- a/go.sum +++ b/go.sum @@ -12,6 +12,8 @@ github.com/allegro/bigcache v1.2.1 h1:hg1sY1raCwic3Vnsvje6TT7/pnZba83LeFck5NrFKS github.com/allegro/bigcache v1.2.1/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -182,6 +184,10 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jrick/logrotate v1.0.0/go.mod h1:LNinyqDIJnpAur+b8yyulnQw/wDuN1+BYKlTRt3OuAQ= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -442,6 +448,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/op-da/cmd/main.go b/op-da/cmd/main.go new file mode 100644 index 0000000..42559c2 --- /dev/null +++ b/op-da/cmd/main.go @@ -0,0 +1,32 @@ +package main + +import ( + "context" + "os" + + "github.com/ethereum-optimism/optimism/op-service/ctxinterrupt" + "github.com/mdehoog/op-nitro/op-da/da" + + "github.com/urfave/cli/v2" + + "github.com/ethereum-optimism/optimism/op-proposer/flags" + "github.com/ethereum-optimism/optimism/op-service/cliapp" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + "github.com/ethereum/go-ethereum/log" +) + +func main() { + oplog.SetupDefaults() + + app := cli.NewApp() + app.Flags = cliapp.ProtectFlags(flags.Flags) + app.Name = "op-da" + app.Usage = "Alt DA server" + app.Action = cliapp.LifecycleCmd(da.Main) + + ctx := ctxinterrupt.WithSignalWaiterMain(context.Background()) + err := app.RunContext(ctx, os.Args) + if err != nil { + log.Crit("Application failed", "message", err) + } +} diff --git a/op-da/da/config.go b/op-da/da/config.go new file mode 100644 index 0000000..dc23273 --- /dev/null +++ b/op-da/da/config.go @@ -0,0 +1,22 @@ +package da + +import ( + "github.com/mdehoog/op-nitro/op-da/flags" + "github.com/urfave/cli/v2" + + oplog "github.com/ethereum-optimism/optimism/op-service/log" +) + +type CLIConfig struct { + Port int + DAURL string + LogConfig oplog.CLIConfig +} + +func NewConfig(ctx *cli.Context) *CLIConfig { + return &CLIConfig{ + Port: ctx.Int(flags.PortFlag.Name), + DAURL: ctx.String(flags.DAURLFlag.Name), + LogConfig: oplog.ReadCLIConfig(ctx), + } +} diff --git a/op-da/da/filestore.go b/op-da/da/filestore.go new file mode 100644 index 0000000..11c3bf1 --- /dev/null +++ b/op-da/da/filestore.go @@ -0,0 +1,30 @@ +package da + +import ( + "context" + "encoding/base64" + "os" + "path/filepath" + + altda "github.com/ethereum-optimism/optimism/op-alt-da" +) + +type filestore struct { + path string +} + +var _ altda.KVStore = &filestore{} + +func NewFilestore(path string) altda.KVStore { + return &filestore{ + path: path, + } +} + +func (s *filestore) Get(ctx context.Context, key []byte) ([]byte, error) { + return os.ReadFile(filepath.Join(s.path, base64.RawURLEncoding.EncodeToString(key))) +} + +func (s *filestore) Put(ctx context.Context, key []byte, value []byte) error { + return os.WriteFile(filepath.Join(s.path, base64.RawURLEncoding.EncodeToString(key)), value, 0644) +} diff --git a/op-da/da/s3store.go b/op-da/da/s3store.go new file mode 100644 index 0000000..8ec5164 --- /dev/null +++ b/op-da/da/s3store.go @@ -0,0 +1,57 @@ +package da + +import ( + "bytes" + "context" + "encoding/base64" + "path/filepath" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + altda "github.com/ethereum-optimism/optimism/op-alt-da" +) + +type s3store struct { + bucket string + path string + downloader *s3manager.Downloader + uploader *s3manager.Uploader +} + +var _ altda.KVStore = &s3store{} + +func NewS3store(bucket, path string) altda.KVStore { + sess := session.Must(session.NewSession()) + downloader := s3manager.NewDownloader(sess) + uploader := s3manager.NewUploader(sess) + return &s3store{ + bucket: bucket, + path: path, + downloader: downloader, + uploader: uploader, + } +} + +func (s *s3store) Get(ctx context.Context, key []byte) ([]byte, error) { + buf := aws.NewWriteAtBuffer([]byte{}) + _, err := s.downloader.DownloadWithContext(ctx, buf, &s3.GetObjectInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filepath.Join(s.path, base64.RawURLEncoding.EncodeToString(key))), + }) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (s *s3store) Put(ctx context.Context, key []byte, value []byte) error { + b := bytes.NewBuffer(value) + _, err := s.uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(s.bucket), + Key: aws.String(filepath.Join(s.path, base64.RawURLEncoding.EncodeToString(key))), + Body: b, + }) + return err +} diff --git a/op-da/da/service.go b/op-da/da/service.go new file mode 100644 index 0000000..fe5251f --- /dev/null +++ b/op-da/da/service.go @@ -0,0 +1,77 @@ +package da + +import ( + "context" + "fmt" + "net/url" + "strings" + "sync/atomic" + + altda "github.com/ethereum-optimism/optimism/op-alt-da" + opservice "github.com/ethereum-optimism/optimism/op-service" + "github.com/ethereum-optimism/optimism/op-service/cliapp" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + "github.com/ethereum/go-ethereum/log" + "github.com/mdehoog/op-nitro/op-da/flags" + "github.com/urfave/cli/v2" +) + +func Main(cliCtx *cli.Context, _ context.CancelCauseFunc) (cliapp.Lifecycle, error) { + cfg := NewConfig(cliCtx) + + l := oplog.NewLogger(oplog.AppOut(cliCtx), cfg.LogConfig) + oplog.SetGlobalLogHandler(l.Handler()) + opservice.ValidateEnvVars(flags.EnvVarPrefix, flags.Flags, l) + + l.Info("Initializing alt-DA server") + return ServiceFromCLIConfig(cliCtx.Context, cfg, l) +} + +func ServiceFromCLIConfig(ctx context.Context, cfg *CLIConfig, l log.Logger) (cliapp.Lifecycle, error) { + store, err := newStore(cfg) + if err != nil { + return nil, err + } + + server := altda.NewDAServer("", cfg.Port, store, l, false) + + return &service{ + server: server, + }, nil +} + +type service struct { + server *altda.DAServer + stopped atomic.Bool +} + +func (s *service) Start(ctx context.Context) error { + return s.server.Start() +} + +func (s *service) Stop(ctx context.Context) error { + if s.stopped.Swap(true) { + return nil + } + return s.server.Stop() +} + +func (s *service) Stopped() bool { + return s.stopped.Load() +} + +func newStore(cfg *CLIConfig) (altda.KVStore, error) { + u, err := url.Parse(cfg.DAURL) + if err != nil { + return nil, fmt.Errorf("failed to parse DA URL: %w", err) + } + switch u.Scheme { + case "s3": + split := strings.SplitN(u.Path, "/", 2) + return NewS3store(split[0], split[1]), nil + case "file": + return NewFilestore(u.Path), nil + default: + return nil, fmt.Errorf("unsupported DA scheme: %s", u.Scheme) + } +} diff --git a/op-da/flags/flags.go b/op-da/flags/flags.go new file mode 100644 index 0000000..8ebd56c --- /dev/null +++ b/op-da/flags/flags.go @@ -0,0 +1,39 @@ +package flags + +import ( + opservice "github.com/ethereum-optimism/optimism/op-service" + oplog "github.com/ethereum-optimism/optimism/op-service/log" + "github.com/urfave/cli/v2" +) + +const EnvVarPrefix = "OP_DA" + +func prefixEnvVar(name string) []string { + return opservice.PrefixEnvVar(EnvVarPrefix, name) +} + +var ( + PortFlag = &cli.IntFlag{ + Name: "port", + Usage: "Port to listen on", + EnvVars: prefixEnvVar("PORT"), + Required: true, + } + DAURLFlag = &cli.StringFlag{ + Name: "da-url", + Usage: "URL for DA (file, http, S3)", + EnvVars: prefixEnvVar("DA_URL"), + Required: true, + } +) + +var requiredFlags = []cli.Flag{ + PortFlag, + DAURLFlag, +} + +func init() { + Flags = append(requiredFlags, oplog.CLIFlags(EnvVarPrefix)...) +} + +var Flags []cli.Flag