From 15863532a5943db20b183003ace423c48ca3a355 Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Sat, 7 Sep 2024 23:01:17 +0000 Subject: [PATCH 1/4] Add ability to process Protobuf Editions files. This should successfully generate code for these files. However, they do not have the correct semantics yet. `protoc` does not translate equivalent features supported in proto2 and proto3 into previous `FieldOptions` fields, and we must adjust the semantics according to what are specified in `FeatureSet` messages. --- proto-lens-tests/package.yaml | 9 ++ proto-lens-tests/proto-lens-tests.cabal | 31 +++++ proto-lens-tests/tests/editions2023.proto | 44 +++++++ proto-lens-tests/tests/editions2023_test.hs | 139 ++++++++++++++++++++ 4 files changed, 223 insertions(+) create mode 100644 proto-lens-tests/tests/editions2023.proto create mode 100644 proto-lens-tests/tests/editions2023_test.hs diff --git a/proto-lens-tests/package.yaml b/proto-lens-tests/package.yaml index 1cf53c66..ef9403d1 100644 --- a/proto-lens-tests/package.yaml +++ b/proto-lens-tests/package.yaml @@ -56,6 +56,15 @@ tests: - Proto.Canonical - Proto.Canonical_Fields + editions2023_test: + main: editions2023_test.hs + source-dirs: tests + dependencies: + - proto-lens-tests + generated-other-modules: + - Proto.Editions2023 + - Proto.Editions2023_Fields + group_test: main: group_test.hs source-dirs: tests diff --git a/proto-lens-tests/proto-lens-tests.cabal b/proto-lens-tests/proto-lens-tests.cabal index 67010eee..b083605e 100644 --- a/proto-lens-tests/proto-lens-tests.cabal +++ b/proto-lens-tests/proto-lens-tests.cabal @@ -234,6 +234,37 @@ test-suite descriptor_test , text default-language: Haskell2010 +test-suite editions2023_test + type: exitcode-stdio-1.0 + main-is: editions2023_test.hs + other-modules: + Paths_proto_lens_tests + Proto.Editions2023 + Proto.Editions2023_Fields + autogen-modules: + Paths_proto_lens_tests + Proto.Editions2023 + Proto.Editions2023_Fields + hs-source-dirs: + tests + build-tool-depends: + proto-lens-protoc:proto-lens-protoc + build-depends: + QuickCheck + , base + , bytestring + , lens-family + , pretty + , proto-lens + , proto-lens-arbitrary + , proto-lens-runtime + , proto-lens-tests + , tasty + , tasty-hunit + , tasty-quickcheck + , text + default-language: Haskell2010 + test-suite enum_test type: exitcode-stdio-1.0 main-is: enum_test.hs diff --git a/proto-lens-tests/tests/editions2023.proto b/proto-lens-tests/tests/editions2023.proto new file mode 100644 index 00000000..ba4546b0 --- /dev/null +++ b/proto-lens-tests/tests/editions2023.proto @@ -0,0 +1,44 @@ +edition = "2023"; + +// Features to preserve proto3 behavior. +// See https://protobuf.dev/editions/features/#proto3-behavior. +// +// Eventually we want this to diverge from proto3.proto, +// e.g., set features on individual fields, +// but we use a test virtually identical to proto3_test.hs for now. +option features.field_presence = IMPLICIT; +option features.enum_type = OPEN; +option features.json_format = ALLOW; +option features.utf8_validation = VERIFY; + +package editions2023; + +message Foo { + int32 a = 1; + repeated string b = 2; + oneof bar { + float c = 3; + bytes d = 4; + Sub s = 8; + } + + message Sub { + int32 e = 1; + } + Sub sub = 5; + + enum FooEnum { + option allow_alias = true; + Enum1 = 0; + Enum2 = 3; + Enum2a = 3; + } + FooEnum enum = 6; + + repeated int32 f = 7; +} + +message Strings { + bytes bytes = 1; + string string = 2; +} diff --git a/proto-lens-tests/tests/editions2023_test.hs b/proto-lens-tests/tests/editions2023_test.hs new file mode 100644 index 00000000..19a89fe6 --- /dev/null +++ b/proto-lens-tests/tests/editions2023_test.hs @@ -0,0 +1,139 @@ +-- Copyright 2024 Google Inc. All Rights Reserved. +-- +-- Use of this source code is governed by a BSD-style +-- license that can be found in the LICENSE file or at +-- https://developers.google.com/open-source/licenses/bsd + +{-# LANGUAGE OverloadedStrings #-} +module Main where + +import Data.ProtoLens +import Lens.Family2 ((&), (.~), (^.)) +import qualified Data.ByteString.Builder as Builder +import Proto.Editions2023 + ( Foo + , Foo'FooEnum(..) + , Foo'Sub + , Strings + ) +import Proto.Editions2023_Fields + ( a + , b + , c + , d + , e + , f + , s + , sub + , maybe'c + , maybe'sub + , maybe's + , enum + , bytes + , string + ) +import Test.Tasty (testGroup) +import Test.Tasty.HUnit (testCase, (@=?), assertBool) + +import Data.ProtoLens.TestUtil + +main :: IO () +main = testMain + [ testGroup "Foo" + [ serializeTo "int32" + (defMessage & a .~ 150 :: Foo) + "a: 150" + $ tagged 1 $ VarInt 150 + , serializeTo "repeated-string" + (defMessage & b .~ ["one", "two"] :: Foo) + (vcat $ map (keyedStr "b") ["one", "two"]) + $ mconcat (map (tagged 2 . Lengthy) ["one", "two"]) + , testGroup "oneof" + [ serializeTo "float" + -- Use denominators that aren't divisible by 2, to fill out the bits. + (defMessage & c .~ (20 / 3) :: Foo) + "c: 6.6666665" + $ tagged 3 $ Fixed32 0x40d55555 + , serializeTo "bytes" + (defMessage & d .~ "a\0b" :: Foo) + "d: \"a\\000b\"" + $ tagged 4 $ Lengthy "a\0b" + , serializeTo "overridden value" + (defMessage & d .~ "a\0b" & c .~ (20 / 3) :: Foo) + "c: 6.6666665" + $ tagged 3 $ Fixed32 0x40d55555 + -- Scalar "oneof" fields should have a "maybe" selector. + , testCase "maybe" $ do + Nothing @=? (defMessage :: Foo) ^. maybe'c + Just 42 @=? ((defMessage :: Foo) & c .~ 42) ^. maybe'c + Nothing @=? (defMessage :: Foo) ^. maybe's + , testCase "message" $ do + Just 42 @=? ((defMessage :: Foo) & s .~ (defMessage :: Foo'Sub) & c .~ 42) ^. maybe'c + Nothing @=? ((defMessage :: Foo) & s .~ (defMessage :: Foo'Sub) & c .~ 42) ^. maybe's + 17 @=? ((defMessage :: Foo) & s . e .~ 17) ^. s . e + let val = (defMessage :: Foo'Sub) & e .~ 17 + Just val @=? ((defMessage :: Foo) & s .~ val) ^. maybe's + ] + -- Repeated scalar fields in proto3 should serialize as "packed" by default. + , serializeTo "packed-by-default" + (defMessage & f .~ [1,2,3] :: Foo) + (vcat [keyedInt "f" x | x <- [1..3]]) + $ tagged 7 $ Lengthy $ mconcat [varInt x | x <- [1..3]] + , runTypedTest (roundTripTest "foo" :: TypedTest Foo) + ] + , testGroup "Strings" + [ deserializeFrom "bytes" + (Just $ defMessage & bytes .~ toStrictByteString invalidUtf8 :: Maybe Strings) + $ tagged 1 $ Lengthy invalidUtf8 + , deserializeFrom "string" + (Nothing :: Maybe Strings) + $ tagged 2 $ Lengthy invalidUtf8 + ] + -- Scalar field defaults are indistinguishable from unset fields. + , testGroup "defaulting" + [ testCase "int" $ (defMessage :: Foo) @=? (defMessage & a .~ 0) + , testCase "bytes" $ (defMessage :: Strings) @=? (defMessage & bytes .~ "") + , testCase "string" $ (defMessage :: Strings) @=? (defMessage & string .~ "") + , testCase "enum" $ (defMessage :: Foo) @=? (defMessage & enum .~ Foo'Enum1) + ] + -- Enums are sum types, except for aliases + , testGroup "enum" + [ testCase "aliases are exported" $ Foo'Enum2 @=? Foo'Enum2a + , serializeTo "serializeTo enum" + (defMessage & enum .~ Foo'Enum2 :: Foo) + "enum: Enum2" + $ tagged 6 $ VarInt 3 + , serializeTo "serializeTo unrecognized" + (defMessage & enum .~ toEnum 9 :: Foo) + "enum: 9" + $ tagged 6 $ VarInt 9 + , testCase "enum values" $ do + map toEnum [0, 3, 3] @=? [Foo'Enum1, Foo'Enum2, Foo'Enum2a] + fromEnum <$> (maybeToEnum 4 :: Maybe Foo'FooEnum) @=? Just 4 + ["Foo'Enum1", "Foo'Enum2", "Foo'Enum2", "Foo'FooEnum'Unrecognized (Foo'FooEnum'UnrecognizedValue 5)"] + @=? map show [Foo'Enum1, Foo'Enum2, Foo'Enum2a, toEnum 5] + ["Enum1", "Enum2", "Enum2", "6"] + @=? map showEnum [Foo'Enum1, Foo'Enum2, Foo'Enum2a, toEnum 6] + [Just Foo'Enum1, Just Foo'Enum2, Just Foo'Enum2, maybeToEnum 4, maybeToEnum 5] + @=? map readEnum ["Enum1", "Enum2", "Enum2a", "4", "5"] + , testCase "enum patterns" $ do + assertBool "enum value" $ case toEnum 3 of + Foo'Enum2 -> True + _ -> False + assertBool "enum alias" $ case toEnum 3 of + Foo'Enum2a -> True + _ -> False + + ] + -- Unset proto3 messages are different than the default value. + , testGroup "submessage" + [ testCase "Nothing" $ Nothing @=? ((defMessage :: Foo) ^. maybe'sub) + , testCase "Just" $ do + let val = (defMessage :: Foo'Sub) & e .~ 3 + Just val @=? ((defMessage :: Foo) & sub .~ val) ^. maybe'sub + ] + ] + + +invalidUtf8 :: Builder.Builder +invalidUtf8 = Builder.word8 0xc3 <> Builder.word8 0x28 From 9e7177a07af0276bc7799ec5f2c99505b5fa8d5b Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Wed, 11 Sep 2024 11:42:07 +0000 Subject: [PATCH 2/4] Accept hypothetical Protobuf Editions files. It will not accept actual Protobuf Editions files by restricting the supported edition to `LEGACY`. This restriction will be lifted once support for edition 2023 is complete. --- proto-lens-tests/package.yaml | 9 -- proto-lens-tests/proto-lens-tests.cabal | 31 ----- proto-lens-tests/tests/editions2023.proto | 44 ------- proto-lens-tests/tests/editions2023_test.hs | 139 -------------------- 4 files changed, 223 deletions(-) delete mode 100644 proto-lens-tests/tests/editions2023.proto delete mode 100644 proto-lens-tests/tests/editions2023_test.hs diff --git a/proto-lens-tests/package.yaml b/proto-lens-tests/package.yaml index ef9403d1..1cf53c66 100644 --- a/proto-lens-tests/package.yaml +++ b/proto-lens-tests/package.yaml @@ -56,15 +56,6 @@ tests: - Proto.Canonical - Proto.Canonical_Fields - editions2023_test: - main: editions2023_test.hs - source-dirs: tests - dependencies: - - proto-lens-tests - generated-other-modules: - - Proto.Editions2023 - - Proto.Editions2023_Fields - group_test: main: group_test.hs source-dirs: tests diff --git a/proto-lens-tests/proto-lens-tests.cabal b/proto-lens-tests/proto-lens-tests.cabal index b083605e..67010eee 100644 --- a/proto-lens-tests/proto-lens-tests.cabal +++ b/proto-lens-tests/proto-lens-tests.cabal @@ -234,37 +234,6 @@ test-suite descriptor_test , text default-language: Haskell2010 -test-suite editions2023_test - type: exitcode-stdio-1.0 - main-is: editions2023_test.hs - other-modules: - Paths_proto_lens_tests - Proto.Editions2023 - Proto.Editions2023_Fields - autogen-modules: - Paths_proto_lens_tests - Proto.Editions2023 - Proto.Editions2023_Fields - hs-source-dirs: - tests - build-tool-depends: - proto-lens-protoc:proto-lens-protoc - build-depends: - QuickCheck - , base - , bytestring - , lens-family - , pretty - , proto-lens - , proto-lens-arbitrary - , proto-lens-runtime - , proto-lens-tests - , tasty - , tasty-hunit - , tasty-quickcheck - , text - default-language: Haskell2010 - test-suite enum_test type: exitcode-stdio-1.0 main-is: enum_test.hs diff --git a/proto-lens-tests/tests/editions2023.proto b/proto-lens-tests/tests/editions2023.proto deleted file mode 100644 index ba4546b0..00000000 --- a/proto-lens-tests/tests/editions2023.proto +++ /dev/null @@ -1,44 +0,0 @@ -edition = "2023"; - -// Features to preserve proto3 behavior. -// See https://protobuf.dev/editions/features/#proto3-behavior. -// -// Eventually we want this to diverge from proto3.proto, -// e.g., set features on individual fields, -// but we use a test virtually identical to proto3_test.hs for now. -option features.field_presence = IMPLICIT; -option features.enum_type = OPEN; -option features.json_format = ALLOW; -option features.utf8_validation = VERIFY; - -package editions2023; - -message Foo { - int32 a = 1; - repeated string b = 2; - oneof bar { - float c = 3; - bytes d = 4; - Sub s = 8; - } - - message Sub { - int32 e = 1; - } - Sub sub = 5; - - enum FooEnum { - option allow_alias = true; - Enum1 = 0; - Enum2 = 3; - Enum2a = 3; - } - FooEnum enum = 6; - - repeated int32 f = 7; -} - -message Strings { - bytes bytes = 1; - string string = 2; -} diff --git a/proto-lens-tests/tests/editions2023_test.hs b/proto-lens-tests/tests/editions2023_test.hs deleted file mode 100644 index 19a89fe6..00000000 --- a/proto-lens-tests/tests/editions2023_test.hs +++ /dev/null @@ -1,139 +0,0 @@ --- Copyright 2024 Google Inc. All Rights Reserved. --- --- Use of this source code is governed by a BSD-style --- license that can be found in the LICENSE file or at --- https://developers.google.com/open-source/licenses/bsd - -{-# LANGUAGE OverloadedStrings #-} -module Main where - -import Data.ProtoLens -import Lens.Family2 ((&), (.~), (^.)) -import qualified Data.ByteString.Builder as Builder -import Proto.Editions2023 - ( Foo - , Foo'FooEnum(..) - , Foo'Sub - , Strings - ) -import Proto.Editions2023_Fields - ( a - , b - , c - , d - , e - , f - , s - , sub - , maybe'c - , maybe'sub - , maybe's - , enum - , bytes - , string - ) -import Test.Tasty (testGroup) -import Test.Tasty.HUnit (testCase, (@=?), assertBool) - -import Data.ProtoLens.TestUtil - -main :: IO () -main = testMain - [ testGroup "Foo" - [ serializeTo "int32" - (defMessage & a .~ 150 :: Foo) - "a: 150" - $ tagged 1 $ VarInt 150 - , serializeTo "repeated-string" - (defMessage & b .~ ["one", "two"] :: Foo) - (vcat $ map (keyedStr "b") ["one", "two"]) - $ mconcat (map (tagged 2 . Lengthy) ["one", "two"]) - , testGroup "oneof" - [ serializeTo "float" - -- Use denominators that aren't divisible by 2, to fill out the bits. - (defMessage & c .~ (20 / 3) :: Foo) - "c: 6.6666665" - $ tagged 3 $ Fixed32 0x40d55555 - , serializeTo "bytes" - (defMessage & d .~ "a\0b" :: Foo) - "d: \"a\\000b\"" - $ tagged 4 $ Lengthy "a\0b" - , serializeTo "overridden value" - (defMessage & d .~ "a\0b" & c .~ (20 / 3) :: Foo) - "c: 6.6666665" - $ tagged 3 $ Fixed32 0x40d55555 - -- Scalar "oneof" fields should have a "maybe" selector. - , testCase "maybe" $ do - Nothing @=? (defMessage :: Foo) ^. maybe'c - Just 42 @=? ((defMessage :: Foo) & c .~ 42) ^. maybe'c - Nothing @=? (defMessage :: Foo) ^. maybe's - , testCase "message" $ do - Just 42 @=? ((defMessage :: Foo) & s .~ (defMessage :: Foo'Sub) & c .~ 42) ^. maybe'c - Nothing @=? ((defMessage :: Foo) & s .~ (defMessage :: Foo'Sub) & c .~ 42) ^. maybe's - 17 @=? ((defMessage :: Foo) & s . e .~ 17) ^. s . e - let val = (defMessage :: Foo'Sub) & e .~ 17 - Just val @=? ((defMessage :: Foo) & s .~ val) ^. maybe's - ] - -- Repeated scalar fields in proto3 should serialize as "packed" by default. - , serializeTo "packed-by-default" - (defMessage & f .~ [1,2,3] :: Foo) - (vcat [keyedInt "f" x | x <- [1..3]]) - $ tagged 7 $ Lengthy $ mconcat [varInt x | x <- [1..3]] - , runTypedTest (roundTripTest "foo" :: TypedTest Foo) - ] - , testGroup "Strings" - [ deserializeFrom "bytes" - (Just $ defMessage & bytes .~ toStrictByteString invalidUtf8 :: Maybe Strings) - $ tagged 1 $ Lengthy invalidUtf8 - , deserializeFrom "string" - (Nothing :: Maybe Strings) - $ tagged 2 $ Lengthy invalidUtf8 - ] - -- Scalar field defaults are indistinguishable from unset fields. - , testGroup "defaulting" - [ testCase "int" $ (defMessage :: Foo) @=? (defMessage & a .~ 0) - , testCase "bytes" $ (defMessage :: Strings) @=? (defMessage & bytes .~ "") - , testCase "string" $ (defMessage :: Strings) @=? (defMessage & string .~ "") - , testCase "enum" $ (defMessage :: Foo) @=? (defMessage & enum .~ Foo'Enum1) - ] - -- Enums are sum types, except for aliases - , testGroup "enum" - [ testCase "aliases are exported" $ Foo'Enum2 @=? Foo'Enum2a - , serializeTo "serializeTo enum" - (defMessage & enum .~ Foo'Enum2 :: Foo) - "enum: Enum2" - $ tagged 6 $ VarInt 3 - , serializeTo "serializeTo unrecognized" - (defMessage & enum .~ toEnum 9 :: Foo) - "enum: 9" - $ tagged 6 $ VarInt 9 - , testCase "enum values" $ do - map toEnum [0, 3, 3] @=? [Foo'Enum1, Foo'Enum2, Foo'Enum2a] - fromEnum <$> (maybeToEnum 4 :: Maybe Foo'FooEnum) @=? Just 4 - ["Foo'Enum1", "Foo'Enum2", "Foo'Enum2", "Foo'FooEnum'Unrecognized (Foo'FooEnum'UnrecognizedValue 5)"] - @=? map show [Foo'Enum1, Foo'Enum2, Foo'Enum2a, toEnum 5] - ["Enum1", "Enum2", "Enum2", "6"] - @=? map showEnum [Foo'Enum1, Foo'Enum2, Foo'Enum2a, toEnum 6] - [Just Foo'Enum1, Just Foo'Enum2, Just Foo'Enum2, maybeToEnum 4, maybeToEnum 5] - @=? map readEnum ["Enum1", "Enum2", "Enum2a", "4", "5"] - , testCase "enum patterns" $ do - assertBool "enum value" $ case toEnum 3 of - Foo'Enum2 -> True - _ -> False - assertBool "enum alias" $ case toEnum 3 of - Foo'Enum2a -> True - _ -> False - - ] - -- Unset proto3 messages are different than the default value. - , testGroup "submessage" - [ testCase "Nothing" $ Nothing @=? ((defMessage :: Foo) ^. maybe'sub) - , testCase "Just" $ do - let val = (defMessage :: Foo'Sub) & e .~ 3 - Just val @=? ((defMessage :: Foo) & sub .~ val) ^. maybe'sub - ] - ] - - -invalidUtf8 :: Builder.Builder -invalidUtf8 = Builder.word8 0xc3 <> Builder.word8 0x28 From bcf5801c3acac7b069905555b7965bd1ac1e36a7 Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Tue, 10 Sep 2024 18:15:45 +0000 Subject: [PATCH 3/4] Build in FeatureSetDefaults for native features. --- .../ProtoLens/Compiler/Editions/Defaults.hs | 44 +++++++++++++++++++ proto-lens-protoc/proto-lens-protoc.cabal | 3 +- 2 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs diff --git a/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs b/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs new file mode 100644 index 00000000..cd3e4345 --- /dev/null +++ b/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs @@ -0,0 +1,44 @@ +-- Copyright 2024 Google LLC. All Rights Reserved. +-- +-- Use of this source code is governed by a BSD-style +-- license that can be found in the LICENSE file or at +-- https://developers.google.com/open-source/licenses/bsd + +{-# LANGUAGE OverloadedLabels #-} + +{-| +Module: Data.ProtoLens.Compiler.Editions.Defaults +Description: Exports defaults for native features in Protobuf Editions. +Copyright: Copyright (c) 2024 Google LLC +License: BSD3 +-} +module Data.ProtoLens.Compiler.Editions.Defaults (defaults) where + +import Data.ByteString (ByteString) +import Data.ProtoLens (decodeMessage) +import Proto.Google.Protobuf.Descriptor (FeatureSetDefaults) + +{-| 'FeatureSetDefaults' message containing feature defaults. + +This contains the defaults from editions @LEGACY@ to @EDITION_2023@ +for the native features defined by @google.protobuf.FeatureSet@. +-} +defaults :: FeatureSetDefaults +defaults | Right m <- msg = m + | Left e <- msg = error $ "unable to decode built-in defaults: " ++ e + where msg = decodeMessage serializedDefaults + +{-| Serialized 'FeatureSetDefaults' message containing feature defaults. + +This contains the defaults from editions @LEGACY@ to @EDITION_2023@ +for the native features defined by @google.protobuf.FeatureSet@. +The message was generated with @protoc@ and translated into a Haskell string: + +> $ protoc --edition_defaults_out=defaults.binpb --edition_defaults_minimum=LEGACY --edition_defaults_maximum=2023 google/protobuf/descriptor.proto +> $ ghci +> ghci> import Data.ByteString as B +> ghci> B.readFile "defaults.binpb" >>= print . show + +-} +serializedDefaults :: ByteString +serializedDefaults = read "\"\\n\\DC3\\CAN\\132\\a\\\"\\NUL*\\f\\b\\SOH\\DLE\\STX\\CAN\\STX \\ETX(\\SOH0\\STX\\n\\DC3\\CAN\\231\\a\\\"\\NUL*\\f\\b\\STX\\DLE\\SOH\\CAN\\SOH \\STX(\\SOH0\\SOH\\n\\DC3\\CAN\\232\\a\\\"\\f\\b\\SOH\\DLE\\SOH\\CAN\\SOH \\STX(\\SOH0\\SOH*\\NUL \\132\\a(\\232\\a\"" diff --git a/proto-lens-protoc/proto-lens-protoc.cabal b/proto-lens-protoc/proto-lens-protoc.cabal index 0d1ac61c..29157018 100644 --- a/proto-lens-protoc/proto-lens-protoc.cabal +++ b/proto-lens-protoc/proto-lens-protoc.cabal @@ -1,6 +1,6 @@ cabal-version: 1.12 --- This file has been generated from package.yaml by hpack version 0.36.0. +-- This file has been generated from package.yaml by hpack version 0.37.0. -- -- see: https://github.com/sol/hpack @@ -42,6 +42,7 @@ executable proto-lens-protoc main-is: protoc-gen-haskell.hs other-modules: Data.ProtoLens.Compiler.Definitions + Data.ProtoLens.Compiler.Editions.Defaults Data.ProtoLens.Compiler.Generate Data.ProtoLens.Compiler.Generate.Commented Data.ProtoLens.Compiler.Generate.Encoding From 029816802f43a9de48ed39a5f904951dead2bbd8 Mon Sep 17 00:00:00 2001 From: Yoo Chung Date: Wed, 11 Sep 2024 00:33:19 +0000 Subject: [PATCH 4/4] Implement feature resolution for Protobuf Editions. --- .../ProtoLens/Compiler/Editions/Defaults.hs | 15 ++-- .../ProtoLens/Compiler/Editions/Features.hs | 86 +++++++++++++++++++ proto-lens-protoc/proto-lens-protoc.cabal | 1 + 3 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Features.hs diff --git a/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs b/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs index cd3e4345..d464f81b 100644 --- a/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs +++ b/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Defaults.hs @@ -12,7 +12,7 @@ Description: Exports defaults for native features in Protobuf Editions. Copyright: Copyright (c) 2024 Google LLC License: BSD3 -} -module Data.ProtoLens.Compiler.Editions.Defaults (defaults) where +module Data.ProtoLens.Compiler.Editions.Defaults (nativeDefaults) where import Data.ByteString (ByteString) import Data.ProtoLens (decodeMessage) @@ -23,10 +23,11 @@ import Proto.Google.Protobuf.Descriptor (FeatureSetDefaults) This contains the defaults from editions @LEGACY@ to @EDITION_2023@ for the native features defined by @google.protobuf.FeatureSet@. -} -defaults :: FeatureSetDefaults -defaults | Right m <- msg = m - | Left e <- msg = error $ "unable to decode built-in defaults: " ++ e - where msg = decodeMessage serializedDefaults +nativeDefaults :: FeatureSetDefaults +nativeDefaults + | Right m <- msg = m + | Left e <- msg = error $ "unable to decode built-in defaults: " ++ e + where msg = decodeMessage serializedNativeDefaults {-| Serialized 'FeatureSetDefaults' message containing feature defaults. @@ -40,5 +41,5 @@ The message was generated with @protoc@ and translated into a Haskell string: > ghci> B.readFile "defaults.binpb" >>= print . show -} -serializedDefaults :: ByteString -serializedDefaults = read "\"\\n\\DC3\\CAN\\132\\a\\\"\\NUL*\\f\\b\\SOH\\DLE\\STX\\CAN\\STX \\ETX(\\SOH0\\STX\\n\\DC3\\CAN\\231\\a\\\"\\NUL*\\f\\b\\STX\\DLE\\SOH\\CAN\\SOH \\STX(\\SOH0\\SOH\\n\\DC3\\CAN\\232\\a\\\"\\f\\b\\SOH\\DLE\\SOH\\CAN\\SOH \\STX(\\SOH0\\SOH*\\NUL \\132\\a(\\232\\a\"" +serializedNativeDefaults :: ByteString +serializedNativeDefaults = read "\"\\n\\DC3\\CAN\\132\\a\\\"\\NUL*\\f\\b\\SOH\\DLE\\STX\\CAN\\STX \\ETX(\\SOH0\\STX\\n\\DC3\\CAN\\231\\a\\\"\\NUL*\\f\\b\\STX\\DLE\\SOH\\CAN\\SOH \\STX(\\SOH0\\SOH\\n\\DC3\\CAN\\232\\a\\\"\\f\\b\\SOH\\DLE\\SOH\\CAN\\SOH \\STX(\\SOH0\\SOH*\\NUL \\132\\a(\\232\\a\"" diff --git a/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Features.hs b/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Features.hs new file mode 100644 index 00000000..9f523a88 --- /dev/null +++ b/proto-lens-protoc/app/Data/ProtoLens/Compiler/Editions/Features.hs @@ -0,0 +1,86 @@ +-- Copyright 2024 Google LLC. All Rights Reserved. +-- +-- Use of this source code is governed by a BSD-style +-- license that can be found in the LICENSE file or at +-- https://developers.google.com/open-source/licenses/bsd + +{-# LANGUAGE OverloadedLabels #-} + +{-| +Module: Data.ProtoLens.Compiler.Editions.Features +Description: Resolves a feature set for a particular edition with Protobuf Editions. +Copyright: Copyright (c) 2024 Google LLC +License: BSD3 +-} +module Data.ProtoLens.Compiler.Editions.Features + ( featuresForEdition + , featuresForEditionFromDefaults + , mergedInto + ) where + +import Control.Applicative ((<|>)) +import Data.ProtoLens (defMessage) +import Data.ProtoLens.Compiler.Editions.Defaults (nativeDefaults) +import Data.ProtoLens.Labels () +import Lens.Family2 ((^.), (.~), (&)) +import Proto.Google.Protobuf.Descriptor + ( Edition + , FeatureSet + , FeatureSetDefaults) + +{-| +Returns the native feature set defaults for the given edition. + +Native features refer to the fields directly defined by 'FeatureSet'. +Features defined as extensions of 'FeatureSet' would be custom features. +-} +featuresForEdition :: Edition -> FeatureSet +featuresForEdition = featuresForEditionFromDefaults nativeDefaults + +{-| +Given the feature set defaults for multiple editions, +return the feature set defaults for the given edition. + +If extensions were supported, this could be used directly +to resolve custom features defined as extensions of 'FeatureSet' +for a particular edition. +-} +featuresForEditionFromDefaults :: FeatureSetDefaults -> Edition -> FeatureSet +featuresForEditionFromDefaults defaults edition + | (d : _) <- candidates = (d ^. #overridableFeatures) `mergedInto` (d ^. #fixedFeatures) + | otherwise = defMessage + where + candidates = dropWhile (\d -> d ^. #edition > edition) recentFirst + + -- #defaults is supposed to be in ascending order of editions + recentFirst = reverse $ defaults ^. #defaults + +{-| +Returns the result of merging a 'FeatureSet' message into another 'FeatureSet' message. + +The semantics are the same as @MergeFrom@ in C++. +When merging a message A into a message B, then any field that has a value in A +will override the value of the same field in B, +otherwise the field in B is used as is. + +Consider using this function as an infix operator. +For example, + +>>> let c = a `mergedInto` b + +could be read like "message C is the message A merged into message B". + +If merging was generally supported for all messages, +we would use it directly instead of using this custom implementation for 'FeatureSet'. +This does not support merging extensions or unknown fields. +-} +mergedInto :: FeatureSet -- ^ Feature set to merge from. + -> FeatureSet -- ^ Feature set to merge into. + -> FeatureSet -- ^ The merged feature set. +mergedInto a b = defMessage + & #maybe'fieldPresence .~ (a ^. #maybe'fieldPresence <|> b ^. #maybe'fieldPresence) + & #maybe'enumType .~ (a ^. #maybe'enumType <|> b ^. #maybe'enumType) + & #maybe'repeatedFieldEncoding .~ (a ^. #maybe'repeatedFieldEncoding <|> b ^. #maybe'repeatedFieldEncoding) + & #maybe'utf8Validation .~ (a ^. #maybe'utf8Validation <|> b ^. #maybe'utf8Validation) + & #maybe'messageEncoding .~ (a ^. #maybe'messageEncoding <|> b ^. #maybe'messageEncoding) + & #maybe'jsonFormat .~ (a ^. #maybe'jsonFormat <|> b ^. #maybe'jsonFormat) diff --git a/proto-lens-protoc/proto-lens-protoc.cabal b/proto-lens-protoc/proto-lens-protoc.cabal index 29157018..e30c5f63 100644 --- a/proto-lens-protoc/proto-lens-protoc.cabal +++ b/proto-lens-protoc/proto-lens-protoc.cabal @@ -43,6 +43,7 @@ executable proto-lens-protoc other-modules: Data.ProtoLens.Compiler.Definitions Data.ProtoLens.Compiler.Editions.Defaults + Data.ProtoLens.Compiler.Editions.Features Data.ProtoLens.Compiler.Generate Data.ProtoLens.Compiler.Generate.Commented Data.ProtoLens.Compiler.Generate.Encoding