diff --git a/experiment/hhfm/hhfm.go b/experiment/hhfm/hhfm.go index c947b33badf9addfc54ae22bf9fefed979630d6d..74c2b6c58dc5c359c2f9fc0a8bf5b8b343bd770d 100644 --- a/experiment/hhfm/hhfm.go +++ b/experiment/hhfm/hhfm.go @@ -116,18 +116,21 @@ func (m Measurer) Run( if helper.Type != "legacy" { return ErrInvalidHelperType } + measurement.TestHelpers = map[string]interface{}{ + "backend": helper.Address, + } // prepare request req, err := http.NewRequest("GeT", helper.Address, nil) if err != nil { return err } headers := map[string]string{ - randx.ChangeCapitalization("Accept"): httpheader.RandomAccept(), + randx.ChangeCapitalization("Accept"): httpheader.Accept(), randx.ChangeCapitalization("Accept-Charset"): "ISO-8859-1,utf-8;q=0.7,*;q=0.3", randx.ChangeCapitalization("Accept-Encoding"): "gzip,deflate,sdch", - randx.ChangeCapitalization("Accept-Language"): httpheader.RandomAcceptLanguage(), + randx.ChangeCapitalization("Accept-Language"): httpheader.AcceptLanguage(), randx.ChangeCapitalization("Host"): randx.Letters(15) + ".com", - randx.ChangeCapitalization("User-Agent"): httpheader.RandomUserAgent(), + randx.ChangeCapitalization("User-Agent"): httpheader.UserAgent(), } for key, value := range headers { // Implementation note: Golang will normalize the header names. We will use diff --git a/experiment/hirl/hirl.go b/experiment/hirl/hirl.go index 9b6491daa69aad9747cb16d2dd15fdecd7f70fc7..a996b9c14637b1a7ffb28cc524e047f40242cc63 100644 --- a/experiment/hirl/hirl.go +++ b/experiment/hirl/hirl.go @@ -93,6 +93,9 @@ func (m Measurer) Run( if helper.Type != "legacy" { return ErrInvalidHelperType } + measurement.TestHelpers = map[string]interface{}{ + "backend": helper.Address, + } out := make(chan MethodResult) for _, method := range m.Methods { callbacks.OnProgress(0.0, fmt.Sprintf("%s...", method.Name())) diff --git a/experiment/tor/tor.go b/experiment/tor/tor.go index 29aa4db41b38d08724536c867d66671ff7edd5da..f037df9641d8c589dd2a08dd8f2a4e5c2d2e7dbf 100644 --- a/experiment/tor/tor.go +++ b/experiment/tor/tor.go @@ -372,15 +372,15 @@ func (rc *resultsCollector) defaultFlexibleConnect( } const snapshotsize = 1 << 8 // no need to include all in report r := oonitemplates.HTTPDo(ctx, oonitemplates.HTTPDoConfig{ - Accept: httpheader.RandomAccept(), - AcceptLanguage: httpheader.RandomAcceptLanguage(), + Accept: httpheader.Accept(), + AcceptLanguage: httpheader.AcceptLanguage(), Beginning: rc.measurement.MeasurementStartTimeSaved, MaxEventsBodySnapSize: snapshotsize, MaxResponseBodySnapSize: snapshotsize, Handler: netxlogger.NewHandler(logger), Method: "GET", URL: url.String(), - UserAgent: httpheader.RandomUserAgent(), + UserAgent: httpheader.UserAgent(), }) tk, err = r.TestKeys, r.Error case "or_port", "or_port_dirauth": diff --git a/experiment/urlgetter/runner.go b/experiment/urlgetter/runner.go index d60326a59b307d1a05dad997958e8c9fa30d3ef7..fa0052016e341ec3a3ff903caebe1f29b0a0adf6 100644 --- a/experiment/urlgetter/runner.go +++ b/experiment/urlgetter/runner.go @@ -52,11 +52,11 @@ func (r Runner) Run(ctx context.Context) error { } } -// MaybeRandomUserAgent returns ua if ua is not empty. Otherwise it +// MaybeUserAgent returns ua if ua is not empty. Otherwise it // returns httpheader.RandomUserAgent(). -func MaybeRandomUserAgent(ua string) string { +func MaybeUserAgent(ua string) string { if ua == "" { - ua = httpheader.RandomUserAgent() + ua = httpheader.UserAgent() } return ua } @@ -66,9 +66,9 @@ func (r Runner) httpGet(ctx context.Context, url string) error { req, err := http.NewRequest(r.Config.Method, url, nil) runtimex.PanicOnError(err, "http.NewRequest failed") req = req.WithContext(ctx) - req.Header.Set("Accept", httpheader.RandomAccept()) - req.Header.Set("Accept-Language", httpheader.RandomAcceptLanguage()) - req.Header.Set("User-Agent", MaybeRandomUserAgent(r.Config.UserAgent)) + req.Header.Set("Accept", httpheader.Accept()) + req.Header.Set("Accept-Language", httpheader.AcceptLanguage()) + req.Header.Set("User-Agent", MaybeUserAgent(r.Config.UserAgent)) if r.Config.HTTPHost != "" { req.Host = r.Config.HTTPHost } diff --git a/experiment/urlgetter/runner_test.go b/experiment/urlgetter/runner_test.go index 9183eedcbb3b1f5d7672d5445da1ca6fd8797f5b..82ec82ac5fd9f4b7f4d2ea6911546a37c28bf774 100644 --- a/experiment/urlgetter/runner_test.go +++ b/experiment/urlgetter/runner_test.go @@ -261,7 +261,7 @@ func TestRunnerWeCanForceUserAgent(t *testing.T) { } func TestRunnerDefaultUserAgent(t *testing.T) { - expected := httpheader.RandomUserAgent() + expected := httpheader.UserAgent() found := atomicx.NewInt64() server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { if r.Header.Get("User-Agent") == expected { diff --git a/go.mod b/go.mod index d932699a79a3a7df18e6755a73c6fb920ee8e36c..13eeb776d77c6ad8de9140649d192f6e24ca55d5 100644 --- a/go.mod +++ b/go.mod @@ -11,12 +11,12 @@ require ( github.com/Psiphon-Labs/goarista v0.0.0-20160825065156-d002785f4c67 // indirect github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 // indirect github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742 // indirect - github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible + github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab // indirect - github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797 // indirect + github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f // indirect github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect - github.com/apex/log v1.4.0 - github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a // indirect + github.com/apex/log v1.6.0 + github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598 // indirect github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f // indirect github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 // indirect github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61 // indirect @@ -31,16 +31,16 @@ require ( github.com/gobwas/glob v0.2.4-0.20180402141543-f00a7392b439 // indirect github.com/google/go-cmp v0.5.0 github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 // indirect + github.com/google/uuid v1.1.1 github.com/gorilla/websocket v1.4.2 github.com/grafov/m3u8 v0.0.0-20171211212457-6ab8f28ed427 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect github.com/iancoleman/strcase v0.0.0-20191112232945-16388991a334 github.com/juju/ratelimit v1.0.2-0.20191002062651-f60b32039441 // indirect github.com/marusama/semaphore v0.0.0-20171214154724-565ffd8e868a // indirect - github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b + github.com/miekg/dns v1.1.30 github.com/montanaflynn/stats v0.6.3 github.com/oschwald/geoip2-golang v1.4.0 - github.com/oschwald/maxminddb-golang v1.7.0 // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 github.com/pion/stun v0.3.5 @@ -58,6 +58,6 @@ require ( go.uber.org/atomic v1.3.3-0.20180806045314-ca680462431f // indirect go.uber.org/multierr v1.1.1-0.20180122172545-ddea229ff1df // indirect go.uber.org/zap v1.9.2-0.20180814183419-67bc79d13d15 // indirect - golang.org/x/net v0.0.0-20200625001655-4c5254603344 - golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 // indirect + golang.org/x/net v0.0.0-20200707034311-ab3426394381 + golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 // indirect ) diff --git a/go.sum b/go.sum index 0735ff1a62bf3ba8aa019e6bf34eae53f276e4e7..3123ba4064caa427edf1694a830db31a4326f457 100644 --- a/go.sum +++ b/go.sum @@ -21,12 +21,12 @@ github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464 h1:VmnMMMheFX github.com/Psiphon-Labs/goptlib v0.0.0-20200406165125-c0e32a7a3464/go.mod h1:Pe5BqN2DdIdChorAXl6bDaQd/wghpCleJfid2NoSli0= github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742 h1:te4lDZfA3tFwaheo+h/GZYGiLGJvm7Dcq2YkFh13QmE= github.com/Psiphon-Labs/net v0.0.0-20191204183604-f5d60dada742/go.mod h1:3mBCrUrPxFCKAhG0ZdEfiU7QU6zl2+gr1HUk1sKYdjI= -github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible h1:+o+Bx2k8rk1PjekH4zUx65TcIinj7611JegDAFfuAvo= -github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200608133602-4698936ee666+incompatible/go.mod h1:VcNEtiQ0z2sCGJf16ZGcpwCas5+r9rt+P20r6LlJ06U= +github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible h1:c76E9yKZiStloyBp1lNFFuGmjf73Kldsk3jpgahE7Mc= +github.com/Psiphon-Labs/psiphon-tunnel-core v2.0.12-0.20200706190114-761b4842e923+incompatible/go.mod h1:VcNEtiQ0z2sCGJf16ZGcpwCas5+r9rt+P20r6LlJ06U= github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab h1:LYp5/y2XR38yyDtNyEZBt0TtFaWgrYMNu3hEcN+om6c= github.com/Psiphon-Labs/quic-go v0.14.1-0.20200306193310-474e74c89fab/go.mod h1:I0Z7XA8KzHZl2MzwqEbZJhBQOHT8ajGUQ5+SWf5KHw0= -github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797 h1:8jTZ+3BhLiYm3QJ4qH2l5Oeb8Vft5eMVG/cYk4uoi4s= -github.com/Psiphon-Labs/tls-tris v0.0.0-20200504183724-16ab4b2ea797/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE= +github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f h1:DZpr9KUNwaL+OjdT7JR+bkr7xXkvikTSMdpt6L1XX6Y= +github.com/Psiphon-Labs/tls-tris v0.0.0-20200610161156-7d791789810f/go.mod h1:v3y9GXFo9Sf2mO6auD2ExGG7oDgrK8TI7eb49ZnUxrE= github.com/Shopify/sarama v1.26.1/go.mod h1:NbSGBSSndYaIhRcBtY9V0U7AyH+x71bG668AuWys/yU= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI= @@ -37,15 +37,15 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/apex/log v1.4.0 h1:jYWeNt9kWJOf1ifht8UjsCQ00eiPnFrUzCBCiiJMw/g= -github.com/apex/log v1.4.0/go.mod h1:UMNC4vQNC7hb5gyr47r18ylK1n34rV7GO+gb0wpXvcE= -github.com/apex/logs v0.0.7/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= +github.com/apex/log v1.6.0 h1:Y50wF1PBIIexIgTm0/7G6gcLitkO5jHK5Mb6wcMY0UI= +github.com/apex/log v1.6.0/go.mod h1:x7s+P9VtvFBXge9Vbn+8TrqKmuzmD35TTkeBHul8UtY= +github.com/apex/logs v1.0.0/go.mod h1:XzxuLZ5myVHDy9SAmYpamKKRNApGj54PfYLcFrXqDwo= github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aristanetworks/fsnotify v1.4.2/go.mod h1:D/rtu7LpjYM8tRJphJ0hUBYpjai8SfX+aSNsWDTq/Ks= github.com/aristanetworks/glog v0.0.0-20191112221043-67e8567f59f3/go.mod h1:KASm+qXFKs/xjSoWn30NrWBBvdTTQq+UjkhjEJHfSFA= -github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a h1:SLrT2O2v0QGaH1JSte1vJ2HtreWZa0MjdGSfXp25qlc= -github.com/aristanetworks/goarista v0.0.0-20200602234848-db8a79a18e4a/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE= +github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598 h1:VbwKXgO1O1JSbI8o3PQqlC/KTem5t3YD7LqvfBT+0Gk= +github.com/aristanetworks/goarista v0.0.0-20200609010056-95bcf8053598/go.mod h1:QZe5Yh80Hp1b6JxQdpfSEEe8X7hTyTEZSosSrFf/oJE= github.com/aristanetworks/splunk-hec-go v0.3.3/go.mod h1:1VHO9r17b0K7WmOlLb9nTk/2YanvOEnLMUgsFrxBROc= github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f h1:SaJ6yqg936TshyeFZqQE+N+9hYkIeL9AMr7S4voCl10= github.com/armon/go-proxyproto v0.0.0-20180202201750-5b7edb60ff5f/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= @@ -130,6 +130,7 @@ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4 h1:OL2d27ueTKnlQJoqLW2fc9pWYulFnJYLWzomGV7HqZo= github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4/go.mod h1:Pw1H1OjSNHiqeuxAduB1BKYXIwFtsyrY47nEqSgEiCM= +github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -186,8 +187,8 @@ github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= -github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b h1:HOmliqRWsjtpSxWaulwlgzShQQCiI0yIaoxJoSDcxao= -github.com/miekg/dns v1.1.30-0.20200428072418-d128d10d176b/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo= +github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= @@ -211,8 +212,6 @@ github.com/oschwald/geoip2-golang v1.4.0 h1:5RlrjCgRyIGDz/mBmPfnAF4h8k0IAcRv9Pvr github.com/oschwald/geoip2-golang v1.4.0/go.mod h1:8QwxJvRImBH+Zl6Aa6MaIcs5YdlZSTKtzmPGzQqi9ng= github.com/oschwald/maxminddb-golang v1.6.0 h1:KAJSjdHQ8Kv45nFIbtoLGrGWqHFajOIm7skTyz/+Dls= github.com/oschwald/maxminddb-golang v1.6.0/go.mod h1:DUJFucBg2cvqx42YmDa/+xHvb0elJtOm3o4aFQ/nb/w= -github.com/oschwald/maxminddb-golang v1.7.0 h1:JmU4Q1WBv5Q+2KZy5xJI+98aUwTIrPPxZUkd5Cwr8Zc= -github.com/oschwald/maxminddb-golang v1.7.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3 h1:YtFkrqsMEj7YqpIhRteVxJxCeC3jJBieuLr0d4C4rSA= @@ -284,6 +283,9 @@ github.com/templexxx/cpufeat v0.0.0-20180724012125-cef66df7f161/go.mod h1:wM7WEv github.com/templexxx/xor v0.0.0-20191217153810-f85b25db303b/go.mod h1:5XA7W9S6mni3h5uvOC75dA3m9CCCaS83lltmc0ukdi4= github.com/tj/assert v0.0.0-20171129193455-018094318fb0 h1:Rw8kxzWo1mr6FSaYXjQELRe88y2KdfynXdnK72rdjtA= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= +github.com/tj/go-buffer v1.0.1/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj52Uc= github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= @@ -340,8 +342,8 @@ golang.org/x/net v0.0.0-20190923162816-aa69164e4478 h1:l5EDrHhldLYb3ZRHDUhXF7Om7 golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200625001655-4c5254603344 h1:vGXIOMxbNfDTk/aXCmfdLgkrSV+Z2tcbze+pEc3v5W4= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -370,8 +372,8 @@ golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c h1:jceGD5YNJGgGMkJz79agzOln1 golang.org/x/sys v0.0.0-20200219091948-cb0a6d8edb6c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd h1:xhmwyvizuTgC2qz7ZlMluP20uW+C3Rm0FD/WLDX8884= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4 h1:5/PjkGUjvEU5Gl6BxmvKRPpqo2uNMv4rcHBMwzk/st8= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666 h1:gVCS+QOncANNPlmlO1AhlU3oxs4V9z+gTtPwIk3p2N8= +golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -427,5 +429,7 @@ gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c h1:grhR+C34yXImVGp7EzNk+DTIk+323eIUWOmEevy6bDo= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/httpheader/accept.go b/internal/httpheader/accept.go index ec757912ab7f70b6bd582f9f8e8eb2325e930eb4..7e90660f8fa5963e4b4633622efbabe9c9ac0465 100644 --- a/internal/httpheader/accept.go +++ b/internal/httpheader/accept.go @@ -1,8 +1,6 @@ package httpheader -// RandomAccept returns a random Accept header. -func RandomAccept() string { - // This is the same that is returned by MK. We have #147 to - // ensure that headers are randomized, if needed. +// Accept returns the Accept header used for measuring. +func Accept() string { return "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8" } diff --git a/internal/httpheader/acceptlanguage.go b/internal/httpheader/acceptlanguage.go index 98cf080d9363c6fc7f968e0c8c4fcc9ba8635033..149efc1d3b76aa53fc4e62c0831f4a143c2a43fb 100644 --- a/internal/httpheader/acceptlanguage.go +++ b/internal/httpheader/acceptlanguage.go @@ -1,8 +1,6 @@ package httpheader -// RandomAcceptLanguage returns a random Accept-Language header -func RandomAcceptLanguage() string { - // This is the same that is returned by MK. We have #147 to - // ensure that headers are randomized, if needed. +// AcceptLanguage returns the Accept-Language header used for measuring. +func AcceptLanguage() string { return "en-US;q=0.8,en;q=0.5" } diff --git a/internal/httpheader/useragent.go b/internal/httpheader/useragent.go index 09d61bda460896e44e7ad8ed9c51cdaf23abbcdc..3649e674c88ece60bd53f46ce3e1391e382e71c0 100644 --- a/internal/httpheader/useragent.go +++ b/internal/httpheader/useragent.go @@ -1,12 +1,9 @@ // Package httpheader contains code to set common HTTP headers. package httpheader -// RandomUserAgent returns a random User-Agent -func RandomUserAgent() string { - // TODO(bassosimone): this user-agent solution is temporary and we - // should instead select one among many user agents. See #147. - // - // 14.9% as of Jun 25, 2020 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ - const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" +// UserAgent returns the User-Agent header used for measuring. +func UserAgent() string { + // 18.2% as of Jul 21, 2020 according to https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ + const ua = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36" return ua } diff --git a/internal/mockable/mockable.go b/internal/mockable/mockable.go index 50f14653855246d82a2f0334877e5c82c13ddce7..2f60d4fb58b817a906b51853b783b0a8d144ec3b 100644 --- a/internal/mockable/mockable.go +++ b/internal/mockable/mockable.go @@ -29,6 +29,7 @@ type ExperimentSession struct { MockableProbeIP string MockableProbeNetworkName string MockableProxyURL *url.URL + MockableResolverIP string MockableSoftwareName string MockableSoftwareVersion string MockableTempDir string @@ -122,6 +123,11 @@ func (sess *ExperimentSession) ProxyURL() *url.URL { return sess.MockableProxyURL } +// ResolverIP implements ExperimentSession.ResolverIP +func (sess *ExperimentSession) ResolverIP() string { + return sess.MockableResolverIP +} + // SoftwareName implements ExperimentSession.SoftwareName func (sess *ExperimentSession) SoftwareName() string { return sess.MockableSoftwareName diff --git a/internal/resources/assets.go b/internal/resources/assets.go index 86df4088bb246a2833e65e0074032ee81c11f1bb..0f21b2153bf5ef0d183c3cbed6287e14ea152d1c 100644 --- a/internal/resources/assets.go +++ b/internal/resources/assets.go @@ -2,7 +2,7 @@ package resources const ( // Version contains the assets version. - Version = 20200619115947 + Version = 20200721121920 // ASNDatabaseName is the ASN-DB file name ASNDatabaseName = "asn.mmdb" @@ -33,18 +33,18 @@ type ResourceInfo struct { // All contains info on all known assets. var All = map[string]ResourceInfo{ "asn.mmdb": { - URLPath: "/ooni/probe-assets/releases/download/20200619115947/asn.mmdb.gz", - GzSHA256: "2bc9e7d0e445d3f93edad094ef9060288850f34652f7592bdedabb5cf74a2065", - SHA256: "d71f0de2fdf17c809e5f60e9b69f874acbe9cde108298ecaedfab16d166cf878", + URLPath: "/ooni/probe-assets/releases/download/20200721121920/asn.mmdb.gz", + GzSHA256: "e0f6adfbbbb565a04b97d9c78d77023e22f8a30477d36acab55169911cb6ba51", + SHA256: "600222c58a464c4cece3096b0b553c0db674276e81e5cd84e76b18acef1c2fce", }, "ca-bundle.pem": { - URLPath: "/ooni/probe-assets/releases/download/20200619115947/ca-bundle.pem.gz", - GzSHA256: "08070cbe24c8895d18bb20ccd746ff7409f1947094a1a47aa59993f588474485", - SHA256: "adf770dfd574a0d6026bfaa270cb6879b063957177a991d453ff1d302c02081f", + URLPath: "/ooni/probe-assets/releases/download/20200721121920/ca-bundle.pem.gz", + GzSHA256: "96de2f6469ce24c1909c82704df519566ec9ecddb7fd8b6ae635dba2a55c0e8c", + SHA256: "726889705b00f736200ed7999f7a50021b8735d53228d679c4e6665aa3b44987", }, "country.mmdb": { - URLPath: "/ooni/probe-assets/releases/download/20200619115947/country.mmdb.gz", - GzSHA256: "a3b13c78c149da4fa8bd63bbb0c25ead9737c1da3bf2e3199f2bca86d73c01b9", - SHA256: "83f369ddcb560862996848b600ce1e5353659dcb9424c5e9dd7f6d980fc56a60", + URLPath: "/ooni/probe-assets/releases/download/20200721121920/country.mmdb.gz", + GzSHA256: "5f23c9155ce6e786c2c94d154d67411f33b39d67f1206fee8d6c61b829acdad5", + SHA256: "cffb80632d05ea76188bacaf6c085bb41540b283b7189b5f5fed9ae7173e7967", }, } diff --git a/model/model.go b/model/model.go index a58aca843bb32188b7b8d4ad8c3e8673051073b6..4d6b749686fa96c9f48e39d3bbb4a1f25428ba83 100644 --- a/model/model.go +++ b/model/model.go @@ -381,6 +381,7 @@ type ExperimentSession interface { ProbeIP() string ProbeNetworkName() string ProxyURL() *url.URL + ResolverIP() string SoftwareName() string SoftwareVersion() string TempDir() string diff --git a/netx/archival/archival.go b/netx/archival/archival.go index 844c4274e3caf781e7a671c3b2498dbbb7663771..40d061da6fdc6718c1f51f304d2686e39a1e022c 100644 --- a/netx/archival/archival.go +++ b/netx/archival/archival.go @@ -59,7 +59,12 @@ var ( ) // TCPConnectStatus contains the TCP connect status. +// +// The Blocked field breaks the separation between measurement and analysis +// we have been enforcing for quite some time now. It is a legacy from the +// Web Connectivity experiment and it should be here because of that. type TCPConnectStatus struct { + Blocked *bool `json:"blocked,omitempty"` // Web Connectivity only Failure *string `json:"failure"` Success bool `json:"success"` } diff --git a/netx/httptransport/system.go b/netx/httptransport/system.go index 064999a16144b149aa3026c3dce0796e137c21a6..d6857f17247fbb56a260875f0ab5537a94996042 100644 --- a/netx/httptransport/system.go +++ b/netx/httptransport/system.go @@ -1,5 +1,3 @@ -// +build go1.14 - package httptransport import ( diff --git a/netx/httptransport/system_compat.go b/netx/httptransport/system_compat.go deleted file mode 100644 index 30a9d943af3c8d3fb769862672e64e2aef1cb613..0000000000000000000000000000000000000000 --- a/netx/httptransport/system_compat.go +++ /dev/null @@ -1,31 +0,0 @@ -// +build !go1.14 - -package httptransport - -import ( - "context" - "net" - "net/http" -) - -// NewSystemTransport creates a new "system" HTTP transport. That is a transport -// using the Go standard library with custom dialer and TLS dialer. -func NewSystemTransport(dialer Dialer, tlsDialer TLSDialer) *http.Transport { - txp := http.DefaultTransport.(*http.Transport).Clone() - txp.DialContext = dialer.DialContext - txp.DialTLS = func(network, address string) (net.Conn, error) { - // Go < 1.14 does not have http.Transport.DialTLSContext - return tlsDialer.DialTLSContext(context.Background(), network, address) - } - // Better for Cloudflare DNS and also better because we have less - // noisy events and we can better understand what happened. - txp.MaxConnsPerHost = 1 - // The following (1) reduces the number of headers that Go will - // automatically send for us and (2) ensures that we always receive - // back the true headers, such as Content-Length. This change is - // functional to OONI's goal of observing the network. - txp.DisableCompression = true - return txp -} - -var _ RoundTripper = &http.Transport{} diff --git a/netx/resolver/dnsoverhttps.go b/netx/resolver/dnsoverhttps.go index 8cf49a2338a81026386c540c105d627b8d50328b..dacb36a65c1b77901a9327feba1cac88ab82c2a5 100644 --- a/netx/resolver/dnsoverhttps.go +++ b/netx/resolver/dnsoverhttps.go @@ -32,7 +32,7 @@ func (t DNSOverHTTPS) RoundTrip(ctx context.Context, query []byte) ([]byte, erro if err != nil { return nil, err } - req.Header.Set("user-agent", httpheader.RandomUserAgent()) + req.Header.Set("user-agent", httpheader.UserAgent()) req.Header.Set("content-type", "application/dns-message") var resp *http.Response resp, err = t.Do(req.WithContext(ctx)) diff --git a/netx/resolver/dnsoverhttps_test.go b/netx/resolver/dnsoverhttps_test.go index 23b5679cde07a5f084ad05a4d9b672ff3a867083..1c0e7226b79f57b15fbc9fe98e0ae694d67e577f 100644 --- a/netx/resolver/dnsoverhttps_test.go +++ b/netx/resolver/dnsoverhttps_test.go @@ -122,7 +122,7 @@ func TestUnitDNSOverHTTPSClientSetsUserAgent(t *testing.T) { var correct bool txp := resolver.DNSOverHTTPS{ Do: func(req *http.Request) (*http.Response, error) { - correct = req.Header.Get("User-Agent") == httpheader.RandomUserAgent() + correct = req.Header.Get("User-Agent") == httpheader.UserAgent() return nil, expected }, URL: "https://doh.powerdns.org/", diff --git a/oonimkall/runner.go b/oonimkall/runner.go index f39859753d8bdd447cafec64a0675f1896059eb6..0df2507f0d7e67122eae8b27c8f9d9c12277d7dd 100644 --- a/oonimkall/runner.go +++ b/oonimkall/runner.go @@ -292,8 +292,6 @@ func (r *runner) Run(ctx context.Context) { endEvent.DownloadedKB = experiment.KibiBytesReceived() endEvent.UploadedKB = experiment.KibiBytesSent() }() - // TODO(bassosimone): here we should recognise the NoEndpoint - // setting used by the WhatsApp experiment. if !r.settings.Options.NoCollector { logger.Info("Opening report... please, be patient") if err := experiment.OpenReport(); err != nil { diff --git a/oonimkall/uuid.go b/oonimkall/uuid.go new file mode 100644 index 0000000000000000000000000000000000000000..70578e6085c27caaf1ccb4e9339a1cdaf85d29be --- /dev/null +++ b/oonimkall/uuid.go @@ -0,0 +1,8 @@ +package oonimkall + +import "github.com/google/uuid" + +// NewUUID4 generates a new UUID4. +func NewUUID4() string { + return uuid.Must(uuid.NewRandom()).String() +} diff --git a/oonimkall/uuid_test.go b/oonimkall/uuid_test.go new file mode 100644 index 0000000000000000000000000000000000000000..5b83e1e3b2729f49c6b7a4330cf540f735704426 --- /dev/null +++ b/oonimkall/uuid_test.go @@ -0,0 +1,13 @@ +package oonimkall_test + +import ( + "testing" + + "github.com/ooni/probe-engine/oonimkall" +) + +func TestNewUUID4(t *testing.T) { + if out := oonimkall.NewUUID4(); len(out) != 36 { + t.Fatal("not the expected output") + } +} diff --git a/session.go b/session.go index 2651eb9d6be3fd91c72c6d0ee7b824a9af030c71..1b205ec58273455680f59f56f4ea554a2d05024a 100644 --- a/session.go +++ b/session.go @@ -466,7 +466,7 @@ func (s *Session) lookupProbeIP(ctx context.Context) (string, error) { return (&iplookup.Client{ HTTPClient: s.DefaultHTTPClient(), Logger: s.logger, - UserAgent: httpheader.RandomUserAgent(), // no need to identify as OONI + UserAgent: httpheader.UserAgent(), // no need to identify as OONI }).Do(ctx) } diff --git a/version/version.go b/version/version.go index 3618469fa1f5b4289f3bce9a0303b372e26e75ff..deba3b46545122bf861af0f5c88fab105a231fcb 100644 --- a/version/version.go +++ b/version/version.go @@ -2,4 +2,4 @@ package version // Version is the version of the engine -const Version = "0.14.0" +const Version = "0.15.0"