diff --git a/.gitignore b/.gitignore index a5df679f7dbe61d183c432af5c49b9341d7aeadf..babedc52dd4b2263b981299431e795dc986ef18d 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ npm-debug.log # we ignore priv/static. You may want to comment # this depending on your deployment strategy. /priv/static/ +/priv/geolix/ # Files matching config/*.secret.exs pattern contain sensitive # data and you should not commit them into version control. diff --git a/config/config.exs b/config/config.exs index 16460450e900bc385fff25137994ab4bd150db36..5c145f1166cd497a2daa21e80396abab39f6c07d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -163,6 +163,15 @@ config :plausible, :custom_domain_server, password: System.get_env("CUSTOM_DOMAIN_SERVER_PASSWORD"), ip: System.get_env("CUSTOM_DOMAIN_SERVER_IP") +config :geolix, + databases: [ + %{ + id: :geolite2_country, + adapter: Geolix.Adapter.MMDB2, + source: "priv/geolix/GeoLite2-Country.mmdb" + } + ] + # Import environment specific config. This must remain at the bottom # of this file so it overrides the configuration defined above. import_config "#{Mix.env()}.exs" diff --git a/config/releases.exs b/config/releases.exs index 2ce95c5a2cf962948a2591fc22361a92b3fd1a96..4b2403238a14ead3be13e42d2ed2296d8096c8a6 100644 --- a/config/releases.exs +++ b/config/releases.exs @@ -51,6 +51,7 @@ cron_enabled = String.to_existing_atom(System.get_env("CRON_ENABLED", "false")) custom_domain_server_ip = System.get_env("CUSTOM_DOMAIN_SERVER_IP") custom_domain_server_user = System.get_env("CUSTOM_DOMAIN_SERVER_USER") custom_domain_server_password = System.get_env("CUSTOM_DOMAIN_SERVER_PASSWORD") +geolite2_country_db = System.get_env("GEOLITE2_COUNTRY_DB") config :plausible, admin_user: admin_user, @@ -171,4 +172,15 @@ config :ref_inspector, config :ua_inspector, init: {Plausible.Release, :configure_ua_inspector} +if geolite2_country_db do + config :geolix, + databases: [ + %{ + id: :country, + adapter: Geolix.Adapter.MMDB2, + source: geolite2_country_db + } + ] +end + config :logger, level: :warn diff --git a/config/test.exs b/config/test.exs index 998a05f3442a0c2b72b1ae2cea644e426c0df509..c84f7d4b7d8f31f41443f82e270dab11df1e1c44 100644 --- a/config/test.exs +++ b/config/test.exs @@ -44,5 +44,14 @@ config :junit_formatter, prepend_project_name?: true, include_filename?: true +config :geolix, + databases: [ + %{ + id: :geolite2_country, + adapter: Geolix.Adapter.Fake, + data: %{{1, 1, 1, 1} => %{country: %{iso_code: "US"}}} + } + ] + config :plausible, session_timeout: 0 diff --git a/lib/plausible_web/controllers/api/external_controller.ex b/lib/plausible_web/controllers/api/external_controller.ex index 5c05bbcf40c0534c40d29ff46f65b23a4ead0906..de56b6538eb5b68e39c82e6e8d2f520a93874824 100644 --- a/lib/plausible_web/controllers/api/external_controller.ex +++ b/lib/plausible_web/controllers/api/external_controller.ex @@ -29,7 +29,6 @@ defmodule PlausibleWeb.Api.ExternalController do defp create_event(conn, params) do uri = params["url"] && URI.parse(params["url"]) - country_code = Plug.Conn.get_req_header(conn, "x-country") |> List.first() user_agent = Plug.Conn.get_req_header(conn, "user-agent") |> List.first() if UAInspector.bot?(user_agent) do @@ -41,6 +40,7 @@ defmodule PlausibleWeb.Api.ExternalController do end ref = parse_referrer(uri, params["referrer"]) + country_code = visitor_country(conn) event_attrs = %{ timestamp: NaiveDateTime.utc_now(), @@ -71,6 +71,28 @@ defmodule PlausibleWeb.Api.ExternalController do end end + defp get_ip(conn) do + forwarded_for = List.first(Plug.Conn.get_req_header(conn, "x-forwarded-for")) + + if forwarded_for do + String.split(forwarded_for, ",") + |> Enum.map(&String.trim/1) + |> List.first + else + to_string(:inet_parse.ntoa(conn.remote_ip)) + end + end + + defp visitor_country(conn) do + result = get_ip(conn) + |> Geolix.lookup() + |> Map.get(:geolite2_country) + + if result do + result.country.iso_code + end + end + defp parse_referrer(_, nil), do: nil defp parse_referrer(uri, referrer_str) do @@ -87,8 +109,7 @@ defmodule PlausibleWeb.Api.ExternalController do |> binary_part(0, 16) user_agent = List.first(Plug.Conn.get_req_header(conn, "user-agent")) || "" - # Netlify sets this header as the remote client IP - ip_address = List.first(Plug.Conn.get_req_header(conn, "x-bb-ip")) || "" + ip_address = get_ip(conn) domain = strip_www(params["domain"]) || "" SipHash.hash!(hash_key, user_agent <> ip_address <> domain) diff --git a/mix.exs b/mix.exs index 52c9ab7883894bb503d6531c54be7a9afbb65104..8e7c03aa97bdca28b9d4aa387a52d25078f23cf0 100644 --- a/mix.exs +++ b/mix.exs @@ -89,6 +89,8 @@ defmodule Plausible.MixProject do {:siphash, "~> 3.2"}, {:oban, "~> 1.2"}, {:sshex, "2.2.1"}, + {:geolix, "~> 1.0"}, + {:geolix_adapter_mmdb2, "~> 0.5.0"}, {:clickhousex, [git: "https://github.com/atlas-forks/clickhousex.git"]} ] end diff --git a/mix.lock b/mix.lock index 2de692209378c3eb47211a8e041e4b8c1aa562ab..0124f72edeb191c08b423ac7d5df0424b2e1be97 100644 --- a/mix.lock +++ b/mix.lock @@ -24,6 +24,8 @@ "excoveralls": {:hex, :excoveralls, "0.12.3", "2142be7cb978a3ae78385487edda6d1aff0e482ffc6123877bb7270a8ffbcfe0", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "568a3e616c264283f5dea5b020783ae40eef3f7ee2163f7a67cbd7b35bcadada"}, "file_system": {:hex, :file_system, "0.2.8", "f632bd287927a1eed2b718f22af727c5aeaccc9a98d8c2bd7bff709e851dc986", [:mix], [], "hexpm", "97a3b6f8d63ef53bd0113070102db2ce05352ecf0d25390eb8d747c2bde98bca"}, "gen_smtp": {:hex, :gen_smtp, "0.15.0", "9f51960c17769b26833b50df0b96123605a8024738b62db747fece14eb2fbfcc", [:rebar3], [], "hexpm", "29bd14a88030980849c7ed2447b8db6d6c9278a28b11a44cafe41b791205440f"}, + "geolix": {:hex, :geolix, "1.1.0", "8b0fe847fef486d9e8b7c21eae6cbc2d998fb249e61d3f4f136f8016b9c1c833", [:mix], [{:poolboy, "~> 1.0", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "980854f2aef30c288dc79e86c5267806d704c4525fde1b75de9a92f67fb16300"}, + "geolix_adapter_mmdb2": {:hex, :geolix_adapter_mmdb2, "0.5.0", "5912723d9538ecddc6b29b1d8041b917b735a78fd3c122bfea8c44aa782e3369", [:mix], [{:geolix, "~> 1.1", [hex: :geolix, repo: "hexpm", optional: false]}, {:mmdb2_decoder, "~> 3.0", [hex: :mmdb2_decoder, repo: "hexpm", optional: false]}], "hexpm", "cb1485b6a0a2d3e541949207428a245718dbf1258453a0df0e5fdd925bcecd3e"}, "gettext": {:hex, :gettext, "0.18.0", "406d6b9e0e3278162c2ae1de0a60270452c553536772167e2d701f028116f870", [:mix], [], "hexpm", "c3f850be6367ebe1a08616c2158affe4a23231c70391050bf359d5f92f66a571"}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm", "e0100f8ef7d1124222c11ad362c857d3df7cb5f4204054f9f0f4a728666591fc"}, "httpoison": {:hex, :httpoison, "1.6.2", "ace7c8d3a361cebccbed19c283c349b3d26991eff73a1eaaa8abae2e3c8089b6", [:mix], [{:hackney, "~> 1.15 and >= 1.15.2", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "aa2c74bd271af34239a3948779612f87df2422c2fdcfdbcec28d9c105f0773fe"}, @@ -35,6 +37,7 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm", "6cbe761d6a0ca5a31a0931bf4c63204bceb64538e664a8ecf784a9a6f3b875f1"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, + "mmdb2_decoder": {:hex, :mmdb2_decoder, "3.0.0", "54828676a36e75e9a25bc9a0bb0598d4c7fcc767bf0b40674850b22e05b7b6cc", [:mix], [], "hexpm", "359dc9242915538d1dceb9f6d96c72201dca76ce62e49d22e2ed1e86f20bea8e"}, "nanoid": {:hex, :nanoid, "2.0.2", "f3f7b4bf103ab6667f22beb00b6315825ee3f30100dd2c93d534e5c02164e857", [:mix], [], "hexpm", "3095cb1fac7bbc78843a8ccd99f1af375d0da1d3ebaa8552e846b73438c0c44f"}, "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm", "9374f4302045321874cccdc57eb975893643bd69c3b22bf1312dab5f06e5788e"}, "oban": {:hex, :oban, "1.2.0", "7cca94d341be43d220571e28f69131c4afc21095b25257397f50973d3fc59b07", [:mix], [{:ecto_sql, "~> 3.1", [hex: :ecto_sql, repo: "hexpm", optional: false]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: false]}, {:postgrex, "~> 0.14", [hex: :postgrex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "ba5f8b3f7d76967b3e23cf8014f6a13e4ccb33431e4808f036709a7f822362ee"}, diff --git a/test/plausible_web/controllers/api/external_controller_test.exs b/test/plausible_web/controllers/api/external_controller_test.exs index ef8b95609442d4300c35ab195a2472d93b393d36..45a7972eaa60b36b4e21824b0b7e07aaa45dd049 100644 --- a/test/plausible_web/controllers/api/external_controller_test.exs +++ b/test/plausible_web/controllers/api/external_controller_test.exs @@ -17,7 +17,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do end @user_agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.110 Safari/537.36" - @country_code "EE" describe "POST /api/event" do test "records the event", %{conn: conn} do @@ -33,7 +32,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do conn |> put_req_header("content-type", "text/plain") |> put_req_header("user-agent", @user_agent) - |> put_req_header("x-country", @country_code) |> post("/api/event", Jason.encode!(params)) pageview = get_event("external-controller-test-1.com") @@ -42,7 +40,6 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert pageview["hostname"] == "gigride.live" assert pageview["domain"] == "external-controller-test-1.com" assert pageview["pathname"] == "/" - assert pageview["country_code"] == @country_code end test "www. is stripped from domain", %{conn: conn} do @@ -363,6 +360,25 @@ defmodule PlausibleWeb.Api.ExternalControllerTest do assert event["name"] == "custom event" end + # Fake data is set up in config/test.exs + test "looks up the country from the ip address", %{conn: conn} do + params = %{ + name: "pageview", + domain: "external-controller-test-19.com", + url: "http://gigride.live/" + } + + conn + |> put_req_header("content-type", "text/plain") + |> put_req_header("x-forwarded-for", "1.1.1.1") + |> post("/api/event", Jason.encode!(params)) + + pageview = get_event("external-controller-test-19.com") + + assert pageview["country_code"] == "US" + end + + test "responds 400 when required fields are missing", %{conn: conn} do params = %{}