From 0fc585e0773a668608da2da6d4f5aef9d5345e22 Mon Sep 17 00:00:00 2001 From: yanghao05 Date: Fri, 12 May 2023 20:02:29 +0800 Subject: [PATCH] add new client --- go.mod | 17 +- go.sum | 42 ++ providers/httpclient/client.go | 151 +++++ providers/httpclient/config.go | 35 + providers/httpclient/cookiejar/jar.go | 704 ++++++++++++++++++++ providers/httpclient/cookiejar/punycode.go | 159 +++++ providers/httpclient/cookiejar/serialize.go | 188 ++++++ 7 files changed, 1294 insertions(+), 2 deletions(-) create mode 100644 providers/httpclient/client.go create mode 100644 providers/httpclient/config.go create mode 100644 providers/httpclient/cookiejar/jar.go create mode 100644 providers/httpclient/cookiejar/punycode.go create mode 100644 providers/httpclient/cookiejar/serialize.go diff --git a/go.mod b/go.mod index dee5970..fd3a030 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/go-redis/redis/v8 v8.11.5 github.com/gofrs/uuid v4.0.0+incompatible github.com/golang-jwt/jwt/v4 v4.4.3 + github.com/imroc/req/v3 v3.34.0 + github.com/juju/go4 v0.0.0-20160222163258-40d72ab9641a github.com/mojocn/base64Captcha v1.3.5 github.com/pkg/errors v0.9.1 github.com/rogeecn/gen v1.0.11 @@ -18,8 +20,10 @@ require ( go.uber.org/dig v1.15.0 go.uber.org/zap v1.21.0 golang.org/x/crypto v0.8.0 + golang.org/x/net v0.9.0 golang.org/x/sync v0.1.0 google.golang.org/grpc v1.52.0 + gopkg.in/retry.v1 v1.0.3 gorm.io/driver/mysql v1.4.1 gorm.io/driver/postgres v1.4.4 gorm.io/gen v0.3.19 @@ -36,10 +40,15 @@ require ( github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.11.2 // indirect github.com/go-sql-driver/mysql v1.6.0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/goccy/go-json v0.10.0 // indirect github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0 // indirect + github.com/golang/mock v1.6.0 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/google/pprof v0.0.0-20230426061923-93006964c1fc // indirect github.com/google/uuid v1.3.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -60,8 +69,12 @@ require ( github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/onsi/gomega v1.27.4 // indirect + github.com/onsi/ginkgo/v2 v2.9.2 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect + github.com/quic-go/qpack v0.4.0 // indirect + github.com/quic-go/qtls-go1-19 v0.3.2 // indirect + github.com/quic-go/qtls-go1-20 v0.2.2 // indirect + github.com/quic-go/quic-go v0.34.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect @@ -71,9 +84,9 @@ require ( github.com/ugorji/go/codec v1.2.9 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.8.0 // indirect + golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 // indirect golang.org/x/image v0.0.0-20190802002840-cff245a6509b // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.9.0 // indirect golang.org/x/sys v0.7.0 // indirect golang.org/x/text v0.9.0 // indirect golang.org/x/tools v0.8.0 // indirect diff --git a/go.sum b/go.sum index aedeb26..cd9b671 100644 --- a/go.sum +++ b/go.sum @@ -85,6 +85,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= @@ -103,6 +104,7 @@ github.com/go-gormigrate/gormigrate/v2 v2.0.2 h1:YV4Lc5yMQX8ahVW0ENPq6sPhrhdkGuk github.com/go-gormigrate/gormigrate/v2 v2.0.2/go.mod h1:vld36QpBTfTzLealsHsmQQJK5lSwJt6wiORv+oFX8/I= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= @@ -115,6 +117,8 @@ github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/goccy/go-json v0.10.0 h1:mXKd9Qw4NuzShiRlOXKews24ufknHO7gx30lsDyokKA= github.com/goccy/go-json v0.10.0/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= @@ -144,6 +148,8 @@ github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -160,9 +166,11 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -188,6 +196,8 @@ github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20230426061923-93006964c1fc h1:AGDHt781oIcL4EFk7cPnvBUYTwU8BEU6GDTO3ZMn1sE= +github.com/google/pprof v0.0.0-20230426061923-93006964c1fc/go.mod h1:79YE0hCXdHag9sBkw2o+N/YnZtTkXi0UT9Nnixa5eYk= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -198,6 +208,11 @@ github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5m github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -205,6 +220,8 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imroc/req/v3 v3.34.0 h1:ChzOrBy397f+mvLTGT0yai9FeODsZnLQU7EnRvVlhiY= +github.com/imroc/req/v3 v3.34.0/go.mod h1:zWuou7ZZtPLp/gmMj58jARgZuhKlEdb2gLxDeNfWpCA= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= @@ -276,6 +293,8 @@ github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnr github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/juju/go4 v0.0.0-20160222163258-40d72ab9641a h1:45JtCyuNYE+QN9aPuR1ID9++BQU+NMTMudHSuaK0Las= +github.com/juju/go4 v0.0.0-20160222163258-40d72ab9641a/go.mod h1:RVHtZuvrpETIepiNUrNlih2OynoFf1eM6DGC6dloXzk= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -326,6 +345,8 @@ github.com/mojocn/base64Captcha v1.3.5/go.mod h1:/tTTXn4WTpX9CfrmipqRytCpJ27Uw3G github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= +github.com/onsi/ginkgo/v2 v2.9.2/go.mod h1:WHcJJG2dIlcCqVfBAwUCrJxSPFb6v4azBwgxeMeDuts= github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= @@ -339,10 +360,20 @@ github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qR github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= +github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= +github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= +github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/quic-go v0.34.0 h1:OvOJ9LFjTySgwOTYUZmNoq0FzVicP8YujpV0kB7m2lU= +github.com/quic-go/quic-go v0.34.0/go.mod h1:+4CVgVppm0FNjpG3UcX8Joi/frKOH7/ciD5yGcwOO1g= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogeecn/gen v1.0.11 h1:POphMIL4Jt+2nNl8OAayGhhBqb3jF3XRpfEO/WS8Byg= github.com/rogeecn/gen v1.0.11/go.mod h1:LEbdAmYd12LM+sfRNfyM4RyThP0VEA1Z+R96OQlQ8aA= +github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a h1:3QH7VyOaaiUHNrA9Se4YQIRkDTCw1EJls9xTUCaCeRM= +github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= @@ -442,6 +473,7 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20221005025214-4161e89ecf1b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.8.0 h1:pd9TJtTueMTVQXzk8E2XESSMQDj/U7OUu0PqJqPXQjQ= +golang.org/x/crypto v0.8.0/go.mod h1:mRqEX+O9/h5TFCrQhkgjo2yKi0yYA+9ecGkdQoHrywE= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -452,6 +484,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53 h1:5llv2sWeaMSnA3w2kS57ouQQ4pudlXrR0dCgw51QK9o= +golang.org/x/exp v0.0.0-20230425010034-47ecfdc1ba53/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190501045829-6d32002ffd75/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b h1:+qEpEAPhDZ1o0x3tHzZTQDArnOixOzGD9HUJfcg0mb4= @@ -479,6 +513,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -517,6 +552,7 @@ golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= +golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -591,6 +627,7 @@ golang.org/x/sys v0.0.0-20220224120231-95c6836cb0e7/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= +golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -603,6 +640,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -659,8 +697,10 @@ golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= +golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -771,6 +811,8 @@ gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:a gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= +gopkg.in/retry.v1 v1.0.3 h1:a9CArYczAVv6Qs6VGoLMio99GEs7kY9UzSF9+LD+iGs= +gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/providers/httpclient/client.go b/providers/httpclient/client.go new file mode 100644 index 0000000..de2a7c3 --- /dev/null +++ b/providers/httpclient/client.go @@ -0,0 +1,151 @@ +package httpclient + +import ( + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/imroc/req/v3" + "github.com/rogeecn/atom/container" + "github.com/rogeecn/atom/providers/httpclient/cookiejar" + "github.com/rogeecn/atom/providers/log" + "github.com/rogeecn/atom/utils/opt" +) + +type Client struct { + client *req.Client + jar *cookiejar.Jar +} + +func Provide(opts ...opt.Option) error { + o := opt.New(opts...) + var config Config + if err := o.UnmarshalConfig(&config); err != nil { + log.Fatal(err) + } + return container.Container.Provide(func() (*Client, error) { + c := &Client{} + + client := req.C() + if config.DevMode { + client.DevMode() + } + + if config.CookieJarFile != "" { + dir := filepath.Dir(config.CookieJarFile) + if _, err := os.Stat(dir); os.IsNotExist(err) { + err = os.MkdirAll(dir, 0755) + if err != nil { + return nil, err + } + } + jar, err := cookiejar.New(&cookiejar.Options{ + Filename: config.CookieJarFile, + }) + if err != nil { + return nil, err + } + + c.jar = jar + client.SetCookieJar(jar) + } + + if config.RootCa != nil { + client.SetRootCertsFromFile(config.RootCa...) + } + + if config.InsecureSkipVerify { + client.EnableInsecureSkipVerify() + } + + if config.UserAgent != "" { + client.SetUserAgent(config.UserAgent) + } + if config.Timeout > 0 { + client.SetTimeout(time.Duration(config.Timeout) * time.Second) + } + + if config.CommonHeaders != nil { + client.SetCommonHeaders(config.CommonHeaders) + } + + if config.AuthBasic.Username != "" && config.AuthBasic.Password != "" { + client.SetCommonBasicAuth(config.AuthBasic.Username, config.AuthBasic.Password) + } + + if config.AuthBearerToken != "" { + client.SetCommonBearerAuthToken(config.AuthBearerToken) + } + + if config.ProxyURL != "" { + client.SetProxyURL(config.ProxyURL) + } + + if config.RedirectPolicy != nil { + client.SetRedirectPolicy(parsePolicies(config.RedirectPolicy)...) + } + + c.client = client + return c, nil + }, o.DiOptions()...) +} + +func parsePolicies(policies []string) []req.RedirectPolicy { + ps := []req.RedirectPolicy{} + for _, policy := range policies { + policyItems := strings.Split(policy, ":") + if len(policyItems) != 2 { + continue + } + + switch policyItems[0] { + case "Max": + max, err := strconv.Atoi(policyItems[1]) + if err != nil { + continue + } + ps = append(ps, req.MaxRedirectPolicy(max)) + case "No": + ps = append(ps, req.NoRedirectPolicy()) + case "SameDomain": + ps = append(ps, req.SameDomainRedirectPolicy()) + case "SameHost": + ps = append(ps, req.SameHostRedirectPolicy()) + case "AllowedHost": + ps = append(ps, req.AllowedHostRedirectPolicy(strings.Split(policyItems[1], ",")...)) + case "AllowedDomain": + ps = append(ps, req.AllowedDomainRedirectPolicy(strings.Split(policyItems[1], ",")...)) + } + } + + return ps +} + +func (c *Client) R() *req.Request { + return c.client.R() +} + +func (c *Client) SaveCookJar() error { + return c.jar.Save() +} + +func (c *Client) GetCookie(key string) (string, bool) { + kv := c.AllCookiesKV() + v, ok := kv[key] + return v, ok +} + +func (c *Client) AllCookies() []*http.Cookie { + return c.jar.AllCookies() +} + +func (c *Client) AllCookiesKV() map[string]string { + return c.jar.KVData() +} +func (c *Client) SetCookie(u *url.URL, cookies []*http.Cookie) { + c.jar.SetCookies(u, cookies) +} diff --git a/providers/httpclient/config.go b/providers/httpclient/config.go new file mode 100644 index 0000000..8e8a8bc --- /dev/null +++ b/providers/httpclient/config.go @@ -0,0 +1,35 @@ +package httpclient + +import ( + "github.com/rogeecn/atom/container" + "github.com/rogeecn/atom/providers/http" + "github.com/rogeecn/atom/utils/opt" +) + +const DefaultPrefix = "HttpClient" + +func DefaultProvider() container.ProviderContainer { + return container.ProviderContainer{ + Provider: Provide, + Options: []opt.Option{ + opt.Prefix(http.DefaultPrefix), + }, + } +} + +type Config struct { + DevMode bool + CookieJarFile string + RootCa []string + UserAgent string + InsecureSkipVerify bool + CommonHeaders map[string]string + Timeout uint + AuthBasic struct { + Username string + Password string + } + AuthBearerToken string + ProxyURL string + RedirectPolicy []string // "Max:10;No;SameDomain;SameHost;AllowedHost:x,x,x,x,x,AllowedDomain:x,x,x,x,x" +} diff --git a/providers/httpclient/cookiejar/jar.go b/providers/httpclient/cookiejar/jar.go new file mode 100644 index 0000000..1208aad --- /dev/null +++ b/providers/httpclient/cookiejar/jar.go @@ -0,0 +1,704 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package cookiejar implements an in-memory RFC 6265-compliant http.CookieJar. +// +// This implementation is a fork of net/http/cookiejar which also +// implements methods for dumping the cookies to persistent +// storage and retrieving them. +package cookiejar + +import ( + "fmt" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "sort" + "strings" + "sync" + "time" + + "github.com/pkg/errors" + "golang.org/x/net/publicsuffix" +) + +// PublicSuffixList provides the public suffix of a domain. For example: +// - the public suffix of "example.com" is "com", +// - the public suffix of "foo1.foo2.foo3.co.uk" is "co.uk", and +// - the public suffix of "bar.pvt.k12.ma.us" is "pvt.k12.ma.us". +// +// Implementations of PublicSuffixList must be safe for concurrent use by +// multiple goroutines. +// +// An implementation that always returns "" is valid and may be useful for +// testing but it is not secure: it means that the HTTP server for foo.com can +// set a cookie for bar.com. +// +// A public suffix list implementation is in the package +// golang.org/x/net/publicsuffix. +type PublicSuffixList interface { + // PublicSuffix returns the public suffix of domain. + // + // TODO: specify which of the caller and callee is responsible for IP + // addresses, for leading and trailing dots, for case sensitivity, and + // for IDN/Punycode. + PublicSuffix(domain string) string + + // String returns a description of the source of this public suffix + // list. The description will typically contain something like a time + // stamp or version number. + String() string +} + +// Options are the options for creating a new Jar. +type Options struct { + // PublicSuffixList is the public suffix list that determines whether + // an HTTP server can set a cookie for a domain. + // + // If this is nil, the public suffix list implementation in golang.org/x/net/publicsuffix + // is used. + PublicSuffixList PublicSuffixList + + // Filename holds the file to use for storage of the cookies. + // If it is empty, the value of DefaultCookieFile will be used. + Filename string + + // NoPersist specifies whether no persistence should be used + // (useful for tests). If this is true, the value of Filename will be + // ignored. + NoPersist bool +} + +// Jar implements the http.CookieJar interface from the net/http package. +type Jar struct { + // filename holds the file that the cookies were loaded from. + filename string + + psList PublicSuffixList + + // mu locks the remaining fields. + mu sync.Mutex + + // entries is a set of entries, keyed by their eTLD+1 and subkeyed by + // their name/domain/path. + entries map[string]map[string]entry +} + +var noOptions Options + +// New returns a new cookie jar. A nil *Options is equivalent to a zero +// Options. +// +// New will return an error if the cookies could not be loaded +// from the file for any reason than if the file does not exist. +func New(o *Options) (*Jar, error) { + return newAtTime(o, time.Now()) +} + +// newAtTime is like New but takes the current time as a parameter. +func newAtTime(o *Options, now time.Time) (*Jar, error) { + jar := &Jar{ + entries: make(map[string]map[string]entry), + } + if o == nil { + o = &noOptions + } + if jar.psList = o.PublicSuffixList; jar.psList == nil { + jar.psList = publicsuffix.List + } + if !o.NoPersist { + if jar.filename = o.Filename; jar.filename == "" { + jar.filename = DefaultCookieFile() + } + if err := jar.load(); err != nil { + return nil, errors.Wrap(err, "cannot load cookies") + } + } + jar.deleteExpired(now) + return jar, nil +} + +// homeDir returns the OS-specific home path as specified in the environment. +func homeDir() string { + if runtime.GOOS == "windows" { + return filepath.Join(os.Getenv("HOMEDRIVE"), os.Getenv("HOMEPATH")) + } + return os.Getenv("HOME") +} + +// entry is the internal representation of a cookie. +// +// This struct type is not used outside of this package per se, but the exported +// fields are those of RFC 6265. +// Note that this structure is marshaled to JSON, so backward-compatibility +// should be preserved. +type entry struct { + Name string + Value string + Domain string + Path string + Secure bool + HttpOnly bool + Persistent bool + HostOnly bool + Expires time.Time + Creation time.Time + LastAccess time.Time + + // Updated records when the cookie was updated. + // This is different from creation time because a cookie + // can be changed without updating the creation time. + Updated time.Time + + // CanonicalHost stores the original canonical host name + // that the cookie was associated with. We store this + // so that even if the public suffix list changes (for example + // when storing/loading cookies) we can still get the correct + // jar keys. + CanonicalHost string +} + +// id returns the domain;path;name triple of e as an id. +func (e *entry) id() string { + return id(e.Domain, e.Path, e.Name) +} + +// id returns the domain;path;name triple as an id. +func id(domain, path, name string) string { + return fmt.Sprintf("%s;%s;%s", domain, path, name) +} + +// shouldSend determines whether e's cookie qualifies to be included in a +// request to host/path. It is the caller's responsibility to check if the +// cookie is expired. +func (e *entry) shouldSend(https bool, host, path string) bool { + return e.domainMatch(host) && e.pathMatch(path) && (https || !e.Secure) +} + +// domainMatch implements "domain-match" of RFC 6265 section 5.1.3. +func (e *entry) domainMatch(host string) bool { + if e.Domain == host { + return true + } + return !e.HostOnly && hasDotSuffix(host, e.Domain) +} + +// pathMatch implements "path-match" according to RFC 6265 section 5.1.4. +func (e *entry) pathMatch(requestPath string) bool { + if requestPath == e.Path { + return true + } + if strings.HasPrefix(requestPath, e.Path) { + if e.Path[len(e.Path)-1] == '/' { + return true // The "/any/" matches "/any/path" case. + } else if requestPath[len(e.Path)] == '/' { + return true // The "/any" matches "/any/path" case. + } + } + return false +} + +// hasDotSuffix reports whether s ends in "."+suffix. +func hasDotSuffix(s, suffix string) bool { + return len(s) > len(suffix) && s[len(s)-len(suffix)-1] == '.' && s[len(s)-len(suffix):] == suffix +} + +type byCanonicalHost struct { + byPathLength +} + +func (s byCanonicalHost) Less(i, j int) bool { + e0, e1 := &s.byPathLength[i], &s.byPathLength[j] + if e0.CanonicalHost != e1.CanonicalHost { + return e0.CanonicalHost < e1.CanonicalHost + } + return s.byPathLength.Less(i, j) +} + +// byPathLength is a []entry sort.Interface that sorts according to RFC 6265 +// section 5.4 point 2: by longest path and then by earliest creation time. +type byPathLength []entry + +func (s byPathLength) Len() int { return len(s) } + +func (s byPathLength) Less(i, j int) bool { + e0, e1 := &s[i], &s[j] + if len(e0.Path) != len(e1.Path) { + return len(e0.Path) > len(e1.Path) + } + if !e0.Creation.Equal(e1.Creation) { + return e0.Creation.Before(e1.Creation) + } + // The following are not strictly necessary + // but are useful for providing deterministic + // behaviour in tests. + if e0.Name != e1.Name { + return e0.Name < e1.Name + } + return e0.Value < e1.Value +} + +func (s byPathLength) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +// Cookies implements the Cookies method of the http.CookieJar interface. +// +// It returns an empty slice if the URL's scheme is not HTTP or HTTPS. +func (j *Jar) Cookies(u *url.URL) (cookies []*http.Cookie) { + return j.cookies(u, time.Now()) +} + +// cookies is like Cookies but takes the current time as a parameter. +func (j *Jar) cookies(u *url.URL, now time.Time) (cookies []*http.Cookie) { + if u.Scheme != "http" && u.Scheme != "https" { + return cookies + } + host, err := canonicalHost(u.Host) + if err != nil { + return cookies + } + key := jarKey(host, j.psList) + + j.mu.Lock() + defer j.mu.Unlock() + + submap := j.entries[key] + if submap == nil { + return cookies + } + + https := u.Scheme == "https" + path := u.Path + if path == "" { + path = "/" + } + + var selected []entry + for id, e := range submap { + if !e.Expires.After(now) { + // Save some space by deleting the value when the cookie + // expires. We can't delete the cookie itself because then + // we wouldn't know that the cookie had expired when + // we merge with another cookie jar. + if e.Value != "" { + e.Value = "" + submap[id] = e + } + continue + } + if !e.shouldSend(https, host, path) { + continue + } + e.LastAccess = now + submap[id] = e + selected = append(selected, e) + } + + sort.Sort(byPathLength(selected)) + for _, e := range selected { + cookies = append(cookies, &http.Cookie{Name: e.Name, Value: e.Value}) + } + + return cookies +} + +// AllCookies returns all cookies in the jar. The returned cookies will +// have Domain, Expires, HttpOnly, Name, Secure, Path, and Value filled +// out. Expired cookies will not be returned. This function does not +// modify the cookie jar. +func (j *Jar) AllCookies() (cookies []*http.Cookie) { + return j.allCookies(time.Now()) +} + +// allCookies is like AllCookies but takes the current time as a parameter. +func (j *Jar) allCookies(now time.Time) []*http.Cookie { + var selected []entry + j.mu.Lock() + defer j.mu.Unlock() + for _, submap := range j.entries { + for _, e := range submap { + if !e.Expires.After(now) { + // Do not return expired cookies. + continue + } + selected = append(selected, e) + } + } + + sort.Sort(byCanonicalHost{byPathLength(selected)}) + cookies := make([]*http.Cookie, len(selected)) + for i, e := range selected { + // Note: The returned cookies do not contain sufficient + // information to recreate the database. + cookies[i] = &http.Cookie{ + Name: e.Name, + Value: e.Value, + Path: e.Path, + Domain: e.Domain, + Expires: e.Expires, + Secure: e.Secure, + HttpOnly: e.HttpOnly, + } + } + + return cookies +} + +// RemoveCookie removes the cookie matching the name, domain and path +// specified by c. +func (j *Jar) RemoveCookie(c *http.Cookie) { + j.mu.Lock() + defer j.mu.Unlock() + id := id(c.Domain, c.Path, c.Name) + key := jarKey(c.Domain, j.psList) + if e, ok := j.entries[key][id]; ok { + e.Value = "" + e.Expires = time.Now().Add(-1 * time.Second) + j.entries[key][id] = e + } +} + +// merge merges all the given entries into j. More recently changed +// cookies take precedence over older ones. +func (j *Jar) merge(entries []entry) { + for _, e := range entries { + if e.CanonicalHost == "" { + continue + } + key := jarKey(e.CanonicalHost, j.psList) + id := e.id() + submap := j.entries[key] + if submap == nil { + j.entries[key] = map[string]entry{ + id: e, + } + continue + } + oldEntry, ok := submap[id] + if !ok || e.Updated.After(oldEntry.Updated) { + submap[id] = e + } + } +} + +var expiryRemovalDuration = 24 * time.Hour + +// deleteExpired deletes all entries that have expired for long enough +// that we can actually expect there to be no external copies of it that +// might resurrect the dead cookie. +func (j *Jar) deleteExpired(now time.Time) { + for tld, submap := range j.entries { + for id, e := range submap { + if !e.Expires.After(now) && !e.Updated.Add(expiryRemovalDuration).After(now) { + delete(submap, id) + } + } + if len(submap) == 0 { + delete(j.entries, tld) + } + } +} + +// RemoveAllHost removes any cookies from the jar that were set for the given host. +func (j *Jar) RemoveAllHost(host string) { + host, err := canonicalHost(host) + if err != nil { + return + } + key := jarKey(host, j.psList) + + j.mu.Lock() + defer j.mu.Unlock() + + expired := time.Now().Add(-1 * time.Second) + submap := j.entries[key] + for id, e := range submap { + if e.CanonicalHost == host { + // Save some space by deleting the value when the cookie + // expires. We can't delete the cookie itself because then + // we wouldn't know that the cookie had expired when + // we merge with another cookie jar. + e.Value = "" + e.Expires = expired + submap[id] = e + } + } +} + +// RemoveAll removes all the cookies from the jar. +func (j *Jar) RemoveAll() { + expired := time.Now().Add(-1 * time.Second) + j.mu.Lock() + defer j.mu.Unlock() + for _, submap := range j.entries { + for id, e := range submap { + // Save some space by deleting the value when the cookie + // expires. We can't delete the cookie itself because then + // we wouldn't know that the cookie had expired when + // we merge with another cookie jar. + e.Value = "" + e.Expires = expired + submap[id] = e + } + } +} + +// SetCookies implements the SetCookies method of the http.CookieJar interface. +// +// It does nothing if the URL's scheme is not HTTP or HTTPS. +func (j *Jar) SetCookies(u *url.URL, cookies []*http.Cookie) { + j.setCookies(u, cookies, time.Now()) +} + +// setCookies is like SetCookies but takes the current time as parameter. +func (j *Jar) setCookies(u *url.URL, cookies []*http.Cookie, now time.Time) { + if len(cookies) == 0 { + return + } + if u.Scheme != "http" && u.Scheme != "https" { + // TODO is this really correct? It might be nice to send + // cookies to websocket connections, for example. + return + } + host, err := canonicalHost(u.Host) + if err != nil { + return + } + key := jarKey(host, j.psList) + defPath := defaultPath(u.Path) + + j.mu.Lock() + defer j.mu.Unlock() + + submap := j.entries[key] + for _, cookie := range cookies { + e, err := j.newEntry(cookie, now, defPath, host) + if err != nil { + continue + } + e.CanonicalHost = host + id := e.id() + if submap == nil { + submap = make(map[string]entry) + j.entries[key] = submap + } + if old, ok := submap[id]; ok { + e.Creation = old.Creation + } else { + e.Creation = now + } + e.Updated = now + e.LastAccess = now + submap[id] = e + } +} + +// canonicalHost strips port from host if present and returns the canonicalized +// host name. +func canonicalHost(host string) (string, error) { + var err error + host = strings.ToLower(host) + if hasPort(host) { + host, _, err = net.SplitHostPort(host) + if err != nil { + return "", err + } + } + if strings.HasSuffix(host, ".") { + // Strip trailing dot from fully qualified domain names. + host = host[:len(host)-1] + } + return toASCII(host) +} + +// hasPort reports whether host contains a port number. host may be a host +// name, an IPv4 or an IPv6 address. +func hasPort(host string) bool { + colons := strings.Count(host, ":") + if colons == 0 { + return false + } + if colons == 1 { + return true + } + return host[0] == '[' && strings.Contains(host, "]:") +} + +// jarKey returns the key to use for a jar. +func jarKey(host string, psl PublicSuffixList) string { + if isIP(host) { + return host + } + + var i int + if psl == nil { + i = strings.LastIndex(host, ".") + if i == -1 { + return host + } + } else { + suffix := psl.PublicSuffix(host) + if suffix == host { + return host + } + i = len(host) - len(suffix) + if i <= 0 || host[i-1] != '.' { + // The provided public suffix list psl is broken. + // Storing cookies under host is a safe stopgap. + return host + } + } + prevDot := strings.LastIndex(host[:i-1], ".") + return host[prevDot+1:] +} + +// isIP reports whether host is an IP address. +func isIP(host string) bool { + return net.ParseIP(host) != nil +} + +// defaultPath returns the directory part of an URL's path according to +// RFC 6265 section 5.1.4. +func defaultPath(path string) string { + if len(path) == 0 || path[0] != '/' { + return "/" // Path is empty or malformed. + } + + i := strings.LastIndex(path, "/") // Path starts with "/", so i != -1. + if i == 0 { + return "/" // Path has the form "/abc". + } + return path[:i] // Path is either of form "/abc/xyz" or "/abc/xyz/". +} + +// newEntry creates an entry from a http.Cookie c. now is the current +// time and is compared to c.Expires to determine deletion of c. defPath +// and host are the default-path and the canonical host name of the URL +// c was received from. +// +// The returned entry should be removed if its expiry time is in the +// past. In this case, e may be incomplete, but it will be valid to call +// e.id (which depends on e's Name, Domain and Path). +// +// A malformed c.Domain will result in an error. +func (j *Jar) newEntry(c *http.Cookie, now time.Time, defPath, host string) (e entry, err error) { + e.Name = c.Name + if c.Path == "" || c.Path[0] != '/' { + e.Path = defPath + } else { + e.Path = c.Path + } + + e.Domain, e.HostOnly, err = j.domainAndType(host, c.Domain) + if err != nil { + return e, err + } + // MaxAge takes precedence over Expires. + if c.MaxAge != 0 { + e.Persistent = true + e.Expires = now.Add(time.Duration(c.MaxAge) * time.Second) + if c.MaxAge < 0 { + return e, nil + } + } else if c.Expires.IsZero() { + e.Expires = endOfTime + } else { + e.Persistent = true + e.Expires = c.Expires + if !c.Expires.After(now) { + return e, nil + } + } + + e.Value = c.Value + e.Secure = c.Secure + e.HttpOnly = c.HttpOnly + + return e, nil +} + +var ( + errIllegalDomain = errors.New("cookiejar: illegal cookie domain attribute") + errMalformedDomain = errors.New("cookiejar: malformed cookie domain attribute") + errNoHostname = errors.New("cookiejar: no host name available (IP only)") +) + +// endOfTime is the time when session (non-persistent) cookies expire. +// This instant is representable in most date/time formats (not just +// Go's time.Time) and should be far enough in the future. +var endOfTime = time.Date(9999, 12, 31, 23, 59, 59, 0, time.UTC) + +// domainAndType determines the cookie's domain and hostOnly attribute. +func (j *Jar) domainAndType(host, domain string) (string, bool, error) { + if domain == "" { + // No domain attribute in the SetCookie header indicates a + // host cookie. + return host, true, nil + } + + if isIP(host) { + // According to RFC 6265 domain-matching includes not being + // an IP address. + // TODO: This might be relaxed as in common browsers. + return "", false, errNoHostname + } + + // From here on: If the cookie is valid, it is a domain cookie (with + // the one exception of a public suffix below). + // See RFC 6265 section 5.2.3. + if domain[0] == '.' { + domain = domain[1:] + } + + if len(domain) == 0 || domain[0] == '.' { + // Received either "Domain=." or "Domain=..some.thing", + // both are illegal. + return "", false, errMalformedDomain + } + domain = strings.ToLower(domain) + + if domain[len(domain)-1] == '.' { + // We received stuff like "Domain=www.example.com.". + // Browsers do handle such stuff (actually differently) but + // RFC 6265 seems to be clear here (e.g. section 4.1.2.3) in + // requiring a reject. 4.1.2.3 is not normative, but + // "Domain Matching" (5.1.3) and "Canonicalized Host Names" + // (5.1.2) are. + return "", false, errMalformedDomain + } + + // See RFC 6265 section 5.3 #5. + if j.psList != nil { + if ps := j.psList.PublicSuffix(domain); ps != "" && !hasDotSuffix(domain, ps) { + if host == domain { + // This is the one exception in which a cookie + // with a domain attribute is a host cookie. + return host, true, nil + } + return "", false, errIllegalDomain + } + } + + // The domain must domain-match host: www.mycompany.com cannot + // set cookies for .ourcompetitors.com. + if host != domain && !hasDotSuffix(host, domain) { + return "", false, errIllegalDomain + } + + return domain, false, nil +} + +// DefaultCookieFile returns the default cookie file to use +// for persisting cookie data. +// The following names will be used in decending order of preference: +// - the value of the $GOCOOKIES environment variable. +// - $HOME/.go-cookies +func DefaultCookieFile() string { + if f := os.Getenv("GOCOOKIES"); f != "" { + return f + } + return filepath.Join(homeDir(), ".go-cookies") +} diff --git a/providers/httpclient/cookiejar/punycode.go b/providers/httpclient/cookiejar/punycode.go new file mode 100644 index 0000000..ea7ceb5 --- /dev/null +++ b/providers/httpclient/cookiejar/punycode.go @@ -0,0 +1,159 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cookiejar + +// This file implements the Punycode algorithm from RFC 3492. + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +// These parameter values are specified in section 5. +// +// All computation is done with int32s, so that overflow behavior is identical +// regardless of whether int is 32-bit or 64-bit. +const ( + base int32 = 36 + damp int32 = 700 + initialBias int32 = 72 + initialN int32 = 128 + skew int32 = 38 + tmax int32 = 26 + tmin int32 = 1 +) + +// encode encodes a string as specified in section 6.3 and prepends prefix to +// the result. +// +// The "while h < length(input)" line in the specification becomes "for +// remaining != 0" in the Go code, because len(s) in Go is in bytes, not runes. +func encode(prefix, s string) (string, error) { + output := make([]byte, len(prefix), len(prefix)+1+2*len(s)) + copy(output, prefix) + delta, n, bias := int32(0), initialN, initialBias + b, remaining := int32(0), int32(0) + for _, r := range s { + if r < 0x80 { + b++ + output = append(output, byte(r)) + } else { + remaining++ + } + } + h := b + if b > 0 { + output = append(output, '-') + } + for remaining != 0 { + m := int32(0x7fffffff) + for _, r := range s { + if m > r && r >= n { + m = r + } + } + delta += (m - n) * (h + 1) + if delta < 0 { + return "", fmt.Errorf("cookiejar: invalid label %q", s) + } + n = m + for _, r := range s { + if r < n { + delta++ + if delta < 0 { + return "", fmt.Errorf("cookiejar: invalid label %q", s) + } + continue + } + if r > n { + continue + } + q := delta + for k := base; ; k += base { + t := k - bias + if t < tmin { + t = tmin + } else if t > tmax { + t = tmax + } + if q < t { + break + } + output = append(output, encodeDigit(t+(q-t)%(base-t))) + q = (q - t) / (base - t) + } + output = append(output, encodeDigit(q)) + bias = adapt(delta, h+1, h == b) + delta = 0 + h++ + remaining-- + } + delta++ + n++ + } + return string(output), nil +} + +func encodeDigit(digit int32) byte { + switch { + case 0 <= digit && digit < 26: + return byte(digit + 'a') + case 26 <= digit && digit < 36: + return byte(digit + ('0' - 26)) + } + panic("cookiejar: internal error in punycode encoding") +} + +// adapt is the bias adaptation function specified in section 6.1. +func adapt(delta, numPoints int32, firstTime bool) int32 { + if firstTime { + delta /= damp + } else { + delta /= 2 + } + delta += delta / numPoints + k := int32(0) + for delta > ((base-tmin)*tmax)/2 { + delta /= base - tmin + k += base + } + return k + (base-tmin+1)*delta/(delta+skew) +} + +// Strictly speaking, the remaining code below deals with IDNA (RFC 5890 and +// friends) and not Punycode (RFC 3492) per se. + +// acePrefix is the ASCII Compatible Encoding prefix. +const acePrefix = "xn--" + +// toASCII converts a domain or domain label to its ASCII form. For example, +// toASCII("bücher.example.com") is "xn--bcher-kva.example.com", and +// toASCII("golang") is "golang". +func toASCII(s string) (string, error) { + if ascii(s) { + return s, nil + } + labels := strings.Split(s, ".") + for i, label := range labels { + if !ascii(label) { + a, err := encode(acePrefix, label) + if err != nil { + return "", err + } + labels[i] = a + } + } + return strings.Join(labels, "."), nil +} + +func ascii(s string) bool { + for i := 0; i < len(s); i++ { + if s[i] >= utf8.RuneSelf { + return false + } + } + return true +} diff --git a/providers/httpclient/cookiejar/serialize.go b/providers/httpclient/cookiejar/serialize.go new file mode 100644 index 0000000..443564a --- /dev/null +++ b/providers/httpclient/cookiejar/serialize.go @@ -0,0 +1,188 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cookiejar + +import ( + "encoding/json" + "io" + "log" + "os" + "path/filepath" + "sort" + "strings" + "time" + + "gopkg.in/retry.v1" + + filelock "github.com/juju/go4/lock" + "github.com/pkg/errors" +) + +// Save saves the cookies to the persistent cookie file. +// Before the file is written, it reads any cookies that +// have been stored from it and merges them into j. +func (j *Jar) Save() error { + if j.filename == "" { + return nil + } + return j.save(time.Now()) +} + +// MarshalJSON implements json.Marshaler by encoding all persistent cookies +// currently in the jar. +func (j *Jar) MarshalJSON() ([]byte, error) { + j.mu.Lock() + defer j.mu.Unlock() + // Marshaling entries can never fail. + data, _ := json.Marshal(j.allPersistentEntries()) + return data, nil +} + +// save is like Save but takes the current time as a parameter. +func (j *Jar) save(now time.Time) error { + locked, err := lockFile(lockFileName(j.filename)) + if err != nil { + return err + } + defer locked.Close() + f, err := os.OpenFile(j.filename, os.O_RDWR|os.O_CREATE, 0600) + if err != nil { + return err + } + defer f.Close() + // TODO optimization: if the file hasn't changed since we + // loaded it, don't bother with the merge step. + + j.mu.Lock() + defer j.mu.Unlock() + if err := j.mergeFrom(f); err != nil { + // The cookie file is probably corrupt. + log.Printf("cannot read cookie file to merge it; ignoring it: %v", err) + } + j.deleteExpired(now) + if err := f.Truncate(0); err != nil { + return errors.Wrap(err, "cannot truncate file") + } + if _, err := f.Seek(0, 0); err != nil { + return err + } + return j.writeTo(f) +} + +// load loads the cookies from j.filename. If the file does not exist, +// no error will be returned and no cookies will be loaded. +func (j *Jar) load() error { + if _, err := os.Stat(filepath.Dir(j.filename)); os.IsNotExist(err) { + // The directory that we'll store the cookie jar + // in doesn't exist, so don't bother trying + // to acquire the lock. + return nil + } + locked, err := lockFile(lockFileName(j.filename)) + if err != nil { + return err + } + defer locked.Close() + f, err := os.Open(j.filename) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + if err := j.mergeFrom(f); err != nil { + return err + } + return nil +} + +// mergeFrom reads all the cookies from r and stores them in the Jar. +func (j *Jar) mergeFrom(r io.Reader) error { + decoder := json.NewDecoder(r) + // Cope with old cookiejar format by just discarding + // cookies, but still return an error if it's invalid JSON. + var data json.RawMessage + if err := decoder.Decode(&data); err != nil { + if err == io.EOF { + // Empty file. + return nil + } + return err + } + var entries []entry + if err := json.Unmarshal(data, &entries); err != nil { + log.Printf("warning: discarding cookies in invalid format (error: %v)", err) + return nil + } + j.merge(entries) + return nil +} + +// writeTo writes all the cookies in the jar to w +// as a JSON array. +func (j *Jar) writeTo(w io.Writer) error { + encoder := json.NewEncoder(w) + entries := j.allPersistentEntries() + if err := encoder.Encode(entries); err != nil { + return err + } + return nil +} + +// allPersistentEntries returns all the entries in the jar, sorted by primarly by canonical host +// name and secondarily by path length. +func (j *Jar) allPersistentEntries() []entry { + var entries []entry + for _, submap := range j.entries { + for _, e := range submap { + if e.Persistent { + entries = append(entries, e) + } + } + } + sort.Sort(byCanonicalHost{entries}) + return entries +} + +func (j *Jar) KVData() map[string]string { + pairs := make(map[string]string) + + entries := j.allPersistentEntries() + if len(entries) == 0 { + return pairs + } + + for _, entry := range entries { + pairs[strings.ToLower(entry.Name)] = entry.Value + } + + return pairs +} + +// lockFileName returns the name of the lock file associated with +// the given path. +func lockFileName(path string) string { + return path + ".lock" +} + +var attempt = retry.LimitTime(3*time.Second, retry.Exponential{ + Initial: 100 * time.Microsecond, + Factor: 1.5, + MaxDelay: 100 * time.Millisecond, +}) + +func lockFile(path string) (io.Closer, error) { + for a := retry.Start(attempt, nil); a.Next(); { + locker, err := filelock.Lock(path) + if err == nil { + return locker, nil + } + if !a.More() { + return nil, errors.Wrap(err, "file locked for too long; giving up") + } + } + panic("unreachable") +}