diff --git a/assets/css/app.css b/assets/css/app.css index a14e269b83a9a3740ab890465aa3d9ccdfc4b7bd..c3d92b8374ba99875c092236b94d133bf1c132dd 100644 --- a/assets/css/app.css +++ b/assets/css/app.css @@ -222,3 +222,21 @@ a { .table-striped tbody tr:nth-child(odd) { background-color: #f1f5f8; } + +.twitter-icon { + width: 1.25em; + height: 1.25em; + display: inline-block; + background-repeat: no-repeat; + background-size: contain; + vertical-align: text-bottom; + background-image: url(data:image/svg+xml;charset=utf-8,%3Csvg%20xmlns%3D%22http%3A%2F%2Fwww.w3.org%2F2000%2Fsvg%22%20viewBox%3D%220%200%2072%2072%22%3E%3Cpath%20fill%3D%22none%22%20d%3D%22M0%200h72v72H0z%22%2F%3E%3Cpath%20class%3D%22icon%22%20fill%3D%22%231da1f2%22%20d%3D%22M68.812%2015.14c-2.348%201.04-4.87%201.744-7.52%202.06%202.704-1.62%204.78-4.186%205.757-7.243-2.53%201.5-5.33%202.592-8.314%203.176C56.35%2010.59%2052.948%209%2049.182%209c-7.23%200-13.092%205.86-13.092%2013.093%200%201.026.118%202.02.338%202.98C25.543%2024.527%2015.9%2019.318%209.44%2011.396c-1.125%201.936-1.77%204.184-1.77%206.58%200%204.543%202.312%208.552%205.824%2010.9-2.146-.07-4.165-.658-5.93-1.64-.002.056-.002.11-.002.163%200%206.345%204.513%2011.638%2010.504%2012.84-1.1.298-2.256.457-3.45.457-.845%200-1.666-.078-2.464-.23%201.667%205.2%206.5%208.985%2012.23%209.09-4.482%203.51-10.13%205.605-16.26%205.605-1.055%200-2.096-.06-3.122-.184%205.794%203.717%2012.676%205.882%2020.067%205.882%2024.083%200%2037.25-19.95%2037.25-37.25%200-.565-.013-1.133-.038-1.693%202.558-1.847%204.778-4.15%206.532-6.774z%22%2F%3E%3C%2Fsvg%3E); +} + +.tweet-text a { + @apply text-blue; +} + +.tweet-text a:hover { + text-decoration: underline; +} diff --git a/assets/js/dashboard/stats/modals/referrer-drilldown.js b/assets/js/dashboard/stats/modals/referrer-drilldown.js index 8ddb0685107ad623652d0010f228b3f5a93cce26..81fc172873df2ccb299ffe93d2153fef9ed87c64 100644 --- a/assets/js/dashboard/stats/modals/referrer-drilldown.js +++ b/assets/js/dashboard/stats/modals/referrer-drilldown.js @@ -1,5 +1,6 @@ import React from "react"; import { Link, withRouter } from 'react-router-dom' +import TweetEmbed from 'react-tweet-embed' import Modal from './modal' import * as api from '../../api' @@ -34,18 +35,69 @@ class ReferrerDrilldownModal extends React.Component { } } - renderReferrer(referrer) { + renderReferrerName(name) { + if (name) { + return <a className="hover:underline" target="_blank" href={'//' + name}>{name}</a> + } else { + return '(no referrer)' + } + } + + renderTweet(tweet, index) { + const authorUrl = `https://twitter.com/${tweet.author_handle}` + const tweetUrl = `${authorUrl}/status/${tweet.tweet_id}` + const border = index === 0 ? '' : ' pt-4 border-t border-grey-light' + return ( - <tr className="text-sm" key={referrer.name}> - <td className="p-2 truncate"> - <a className="hover:underline" target="_blank" href={'//' + referrer.name}>{ referrer.name }</a> - </td> - <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.count)}</td> - {this.showBounceRate() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> } - </tr> + <div key={tweet.tweet_id}> + <div className={"flex items-center my-4" + border} > + <a className="flex items-center group" href={authorUrl} target="_blank"> + <img className="rounded-full w-6" src={tweet.author_image} /> + <div className="font-bold ml-2 group-hover:text-blue">{tweet.author_name}</div> + <div className="ml-2 text-xs text-grey-dark">@{tweet.author_handle}</div> + </a> + <a className="ml-auto twitter-icon" href={tweetUrl} target="_blank"></a> + </div> + <div className="my-2 cursor-text tweet-text" dangerouslySetInnerHTML={{__html: tweet.text}}> + </div> + <div className="text-xs text-grey-darker font-medium"> + {tweet.created} + </div> + </div> ) } + renderReferrer(referrer) { + if (false && referrer.tweets) { + return ( + <tr className="text-sm" key={referrer.name}> + <td className="p-2"> + { this.renderReferrerName(referrer.name) } + <span className="text-grey-dark ml-2 text-xs"> + appears in {referrer.tweets.length} tweets + <svg className="feather ml-1"><use xlinkHref="#feather-chevron-down" /></svg> + </span> + <div className="my-4 ml-4"> + { referrer.tweets.map(this.renderTweet) } + </div> + </td> + <td className="p-2 w-32 font-medium" align="right" valign="top">{numberFormatter(referrer.count)}</td> + {this.showBounceRate() && <td className="p-2 w-32 font-medium" align="right" valign="top">{this.formatBounceRate(referrer)}</td> } + </tr> + ) + } else { + return ( + <tr className="text-sm" key={referrer.name}> + <td className="p-2 truncate"> + { this.renderReferrerName(referrer.name) } + </td> + <td className="p-2 w-32 font-medium" align="right">{numberFormatter(referrer.count)}</td> + {this.showBounceRate() && <td className="p-2 w-32 font-medium" align="right">{this.formatBounceRate(referrer)}</td> } + </tr> + ) + } + } + renderGoalText() { if (this.state.query.filters.goal) { return ( diff --git a/assets/package-lock.json b/assets/package-lock.json index de73f62495bf962f85d3e5b57c9f20e52c84d702..1b38f921adf6743b77a9da8b9310bda4a7f3fd55 100644 --- a/assets/package-lock.json +++ b/assets/package-lock.json @@ -8033,6 +8033,11 @@ "tiny-warning": "^1.0.0" } }, + "react-tweet-embed": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/react-tweet-embed/-/react-tweet-embed-1.2.2.tgz", + "integrity": "sha512-Y932BlSaJsDUsKDucC2opzzd+uhc0YNhrlTa/4Beb2be1od+AjLGo6Fhuo2wPT0D+fF4VTXOyoZyA8Yc88RdYA==" + }, "read-file-stdin": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/read-file-stdin/-/read-file-stdin-0.2.1.tgz", diff --git a/assets/package.json b/assets/package.json index 3aabe4c1c02b550a985df4a62828052a23ca9567..792a616dc982ffca129744dc422fb6cd42218af3 100644 --- a/assets/package.json +++ b/assets/package.json @@ -13,6 +13,7 @@ "react": "^16.11.0", "react-dom": "^16.11.0", "react-router-dom": "^5.1.2", + "react-tweet-embed": "^1.2.2", "url-search-params-polyfill": "^7.0.0" }, "devDependencies": { diff --git a/lib/mix/tasks/fetch_tweets.ex b/lib/mix/tasks/fetch_tweets.ex new file mode 100644 index 0000000000000000000000000000000000000000..42e678dea6ff5ea8eb7541e45827a5a4411f57cd --- /dev/null +++ b/lib/mix/tasks/fetch_tweets.ex @@ -0,0 +1,62 @@ +defmodule Mix.Tasks.FetchTweets do + use Plausible.Repo + alias Plausible.Twitter.Tweet + @oauth_credentials Application.get_env(:plausible, :twitter, %{}) |> OAuther.credentials() + + def run(_args) do + Application.ensure_all_started(:plausible) + execute() + end + + def execute() do + new_links = Repo.all( + from e in Plausible.Event, + where: e.timestamp > fragment("(now() - '6 days'::interval)") and e.timestamp < fragment("(now() - '5 days'::interval)"), + or_where: e.timestamp > fragment("(now() - '1 days'::interval)"), + where: e.referrer_source == "Twitter", + where: e.referrer not in ["t.co", "t.co/"], + distinct: true, + select: e.referrer + ) + + for link <- new_links do + results = search(link) + + for tweet <- results do + {:ok, created} = Timex.parse(tweet["created_at"], "{WDshort} {Mshort} {D} {ISOtime} {Z} {YYYY}") + + Tweet.changeset(%Tweet{}, %{ + link: link, + tweet_id: tweet["id_str"], + author_handle: tweet["user"]["screen_name"], + author_name: tweet["user"]["name"], + author_image: tweet["user"]["profile_image_url"], + text: html_body(tweet), + created: created + }) |> Repo.insert!(on_conflict: :nothing) + end + end + end + + def html_body(tweet) do + body = Enum.reduce(tweet["entities"]["urls"], tweet["full_text"], fn url, text -> + html = "<a href=\"#{url["url"]}\" target=\"_blank\">#{url["display_url"]}</a>" + String.replace(text, url["url"], html) + end) + + Enum.reduce(tweet["entities"]["user_mentions"], body, fn mention, text -> + link = "https://twitter.com/#{mention["screen_name"]}" + html = "<a href=\"#{link}\" target=\"_blank\">@#{mention["screen_name"]}</a>" + String.replace(text, "@" <> mention["screen_name"], html) + end) + end + + defp search(link) do + params = [{"count", 5}, {"tweet_mode", "extended"}, {"q", "https://#{link} -filter:retweets"}] + params = OAuther.sign("get", "https://api.twitter.com/1.1/search/tweets.json", params, @oauth_credentials) + uri = "https://api.twitter.com/1.1/search/tweets.json?" <> URI.encode_query(params) + response = HTTPoison.get!(uri) + Jason.decode!(response.body) + |> Map.get("statuses") + end +end diff --git a/lib/plausible/stats/stats.ex b/lib/plausible/stats/stats.ex index 66cc0b684af878fd08fe44c57c550b72ac6679a9..b17c6696ce18a9133c63f5676611520b74d9cc7b 100644 --- a/lib/plausible/stats/stats.ex +++ b/lib/plausible/stats/stats.ex @@ -228,7 +228,8 @@ defmodule Plausible.Stats do end def referrer_drilldown(site, query, referrer, include \\ []) do - referring_urls = Repo.all(from e in base_query(site, query), + referring_urls = Repo.all( + from e in base_query(site, query), select: %{name: e.referrer, count: count(e.user_id, :distinct)}, group_by: e.referrer, where: e.referrer_source == ^referrer, @@ -236,7 +237,7 @@ defmodule Plausible.Stats do limit: 100 ) - if "bounce_rate" in include do + referring_urls = if "bounce_rate" in include do bounce_rates = bounce_rates_by_referring_url(site, query, Enum.map(referring_urls, fn ref -> ref[:name] end)) Enum.map(referring_urls, fn url -> @@ -245,6 +246,24 @@ defmodule Plausible.Stats do else referring_urls end + + if referrer == "Twitter" do + urls = Enum.map(referring_urls, &(&1[:name])) + + tweets = Repo.all( + from t in Plausible.Twitter.Tweet, + where: t.link in ^urls + ) |> Enum.reduce(%{}, fn tweet, acc -> + Map.update(acc, tweet.link, [tweet], &([tweet | &1])) + end) + |> IO.inspect + + Enum.map(referring_urls, fn url -> + Map.put(url, :tweets, tweets[url[:name]]) + end) + else + referring_urls + end end defp bounce_rates_by_referring_url(site, query, referring_urls) do diff --git a/lib/plausible/twitter/tweet.ex b/lib/plausible/twitter/tweet.ex new file mode 100644 index 0000000000000000000000000000000000000000..8570768aacf5a85a5bd4a95f3e1a89fce8f94173 --- /dev/null +++ b/lib/plausible/twitter/tweet.ex @@ -0,0 +1,26 @@ +defmodule Plausible.Twitter.Tweet do + use Ecto.Schema + import Ecto.Changeset + + @required_fields [:link, :tweet_id, :author_handle, :author_name, :author_image, :text, :created] + + @derive {Jason.Encoder, only: @required_fields} + schema "tweets" do + field :link, :string + + field :tweet_id, :string + field :author_handle, :string + field :author_name, :string + field :author_image, :string + field :text, :string + field :created, :naive_datetime, null: false + + timestamps() + end + + def changeset(tweet, attrs) do + tweet + |> cast(attrs, @required_fields) + |> validate_required(@required_fields) + end +end diff --git a/mix.exs b/mix.exs index 2d5486b2d777745e5822c11b70d549f504921b6f..f2889af83da0fe675180177b693c8b20334fb893 100644 --- a/mix.exs +++ b/mix.exs @@ -59,7 +59,8 @@ defmodule Plausible.MixProject do {:excoveralls, "~> 0.10", only: :test}, {:joken, "~> 2.0"}, {:php_serializer, "~> 0.9.0"}, - {:csv, "~> 2.3"} + {:csv, "~> 2.3"}, + {:oauther, "~> 1.1"} ] end diff --git a/mix.lock b/mix.lock index 22b89cbee23225b8c4215be843a5463c0ef7c315..0d06ce4ee3e1f145e10a572e7eb3207eb678a8ba 100644 --- a/mix.lock +++ b/mix.lock @@ -20,6 +20,7 @@ "elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm"}, "ex_machina": {:hex, :ex_machina, "2.3.0", "92a5ad0a8b10ea6314b876a99c8c9e3f25f4dde71a2a835845b136b9adaf199a", [:mix], [{:ecto, "~> 2.2 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}, {:ecto_sql, "~> 3.0", [hex: :ecto_sql, repo: "hexpm", optional: true]}], "hexpm"}, "excoveralls": {:hex, :excoveralls, "0.12.0", "50e17a1b116fdb7facc2fe127a94db246169f38d7627b391376a0bc418413ce1", [:mix], [{:hackney, "~> 1.0", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm"}, + "extwitter": {:hex, :extwitter, "0.11.0", "9472e19f1711bc60bc7efa594353164532475d7c47ea9f1bb66d4faa889b079e", [:mix], [{:oauther, "~> 1.1", [hex: :oauther, repo: "hexpm", optional: false]}, {:poison, "~> 3.0", [hex: :poison, repo: "hexpm", optional: false]}], "hexpm"}, "file_system": {:hex, :file_system, "0.2.6", "fd4dc3af89b9ab1dc8ccbcc214a0e60c41f34be251d9307920748a14bf41f1d3", [:mix], [], "hexpm"}, "gettext": {:hex, :gettext, "0.17.1", "8baab33482df4907b3eae22f719da492cee3981a26e649b9c2be1c0192616962", [:mix], [], "hexpm"}, "hackney": {:hex, :hackney, "1.15.2", "07e33c794f8f8964ee86cebec1a8ed88db5070e52e904b8f12209773c1036085", [:rebar3], [{:certifi, "2.5.1", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "6.0.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "1.0.1", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "1.1.5", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}], "hexpm"}, @@ -31,6 +32,7 @@ "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm"}, "mime": {:hex, :mime, "1.3.1", "30ce04ab3175b6ad0bdce0035cba77bba68b813d523d1aac73d9781b4d193cf8", [:mix], [], "hexpm"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm"}, + "oauther": {:hex, :oauther, "1.1.1", "7d8b16167bb587ecbcddd3f8792beb9ec3e7b65c1f8ebd86b8dd25318d535752", [:mix], [], "hexpm"}, "parallel_stream": {:hex, :parallel_stream, "1.0.6", "b967be2b23f0f6787fab7ed681b4c45a215a81481fb62b01a5b750fa8f30f76c", [:mix], [], "hexpm"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm"}, "phoenix": {:hex, :phoenix, "1.4.0", "56fe9a809e0e735f3e3b9b31c1b749d4b436e466d8da627b8d82f90eaae714d2", [:mix], [{:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:phoenix_pubsub, "~> 1.1", [hex: :phoenix_pubsub, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 1.0 or ~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: true]}], "hexpm"}, diff --git a/priv/repo/migrations/20200114131538_add_tweets.exs b/priv/repo/migrations/20200114131538_add_tweets.exs new file mode 100644 index 0000000000000000000000000000000000000000..c9cbce9fece4a4d900ee8bb0e9f05cbe50562343 --- /dev/null +++ b/priv/repo/migrations/20200114131538_add_tweets.exs @@ -0,0 +1,20 @@ +defmodule Plausible.Repo.Migrations.AddTweets do + use Ecto.Migration + + def change do + create table(:tweets) do + add :tweet_id, :text, null: false + add :text, :text, null: false + add :author_handle, :text, null: false + add :author_name, :text, null: false + add :author_image, :text, null: false + add :created, :naive_datetime, null: false + add :link, :string, null: false + + timestamps() + end + + create index(:tweets, :link) + create unique_index(:tweets, [:link, :tweet_id]) + end +end diff --git a/test/mix/tasks/fetch_tweets_test.exs b/test/mix/tasks/fetch_tweets_test.exs new file mode 100644 index 0000000000000000000000000000000000000000..dfbe39947237950ecc09374fa6e8cd13606cada1 --- /dev/null +++ b/test/mix/tasks/fetch_tweets_test.exs @@ -0,0 +1,40 @@ +defmodule Mix.Tasks.FetchTweetsTest do + use Plausible.DataCase + alias Mix.Tasks.FetchTweets + + describe "processing tweet entities" do + test "inlines links to the body" do + tweet = %{ + "full_text" => "asd https://t.co/somelink", + "entities" => %{ + "user_mentions" => [], + "urls" => [%{ + "display_url" => "plausible.io", + "indices" => [4, 17], + "url" => "https://t.co/somelink" + }] + } + } + body = FetchTweets.html_body(tweet) + + assert body == "asd <a href=\"https://t.co/somelink\" target=\"_blank\">plausible.io</a>" + end + + test "inlines user mentions to the body" do + tweet = %{ + "full_text" => "asd @hello", + "entities" => %{ + "user_mentions" => [%{ + "screen_name" => "hello", + "id_str" => "123123" + }], + "urls" => [] + } + } + body = FetchTweets.html_body(tweet) + + assert body == "asd <a href=\"https://twitter.com/hello\" target=\"_blank\">@hello</a>" + end + end + +end