Remove StreamAPI

Author: joshsmith

lib/code_corps/github/api/api.ex

@@ -14,8 +14,7 @@ defmodule CodeCorps.GitHub.API do
     |> marshall_response()
   end
 
-  defdelegate eager_get_all(url, headers, opts), to: CodeCorps.GitHub.EagerAPI
-  defdelegate lazy_get_all(url, headers, opts), to: CodeCorps.GitHub.StreamAPI
+  defdelegate get_all(url, headers, opts), to: CodeCorps.GitHub.EagerAPI
 
   @doc """
   Get access token headers for a given `CodeCorps.User` and

lib/code_corps/github/api/eager_api.ex

@@ -1,16 +1,10 @@
 defmodule CodeCorps.GitHub.EagerAPI do
   @moduledoc """
-  This module attempts to implement eager loading of a resource, by trying to
-  fetch all of its pages in parallel.
-
-  This should technically be faster than lazy loading. However, it fails due to
-  timeout errors, even when loading just two pages.
-
-  The assumption is that hackney needs to be configured to allow multiple
-  requests.
+  Eager loads a resource from the GitHub API by fetching all of its pages in
+  parallel.
   """
 
-  def eager_get_all(url, headers, options) do
+  def get_all(url, headers, options) do
     HTTPoison.start
     {:ok, response} = HTTPoison.get(url, headers, options)
 
@@ -43,43 +37,47 @@ defmodule CodeCorps.GitHub.EagerAPI do
   end
 
   defp extract_total_pages(links_string) do
-    # Unfortunately, the paginating info we get from GitHub's responses is not
-    # suitable for easy extraction.
+    # We use regex to parse the pagination info from the GitHub API response
+    # headers.
     #
-    # The information is stored in the following response header:
+    # The headers render pages in the following format:
     #
     # ```
     # {"Link", '<https://api.github.com/search/code?q=addClass+user%3Amozilla&page=15>; rel="next",
     #           <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=34>; rel="last",
     #           <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=1>; rel="first",
     #           <https://api.github.com/search/code?q=addClass+user%3Amozilla&page=13>; rel="prev"'
-    #
     # ```
     #
-    # If the response has no list header, then that means we got all the records
-    # and there's just that one page.
+    # If the response has no list header, then we have received all the records
+    # from the only possible page.
     #
     # If the response has a list header, the value will contain at least the
     # "last" relation.
-    #
-    # Unfortunatly, the only way to parse it is via regex.
     links_string
     |> String.split(", ")
     |> Enum.map(fn link ->
-      # Searches for `rel=`
-      rel = Regex.run(~r{rel="([a-z]+)"}, link) |> List.last
-      # Searches for the following variations:
-      # ```
-      # ?page={match}>
-      # ?page={match}&...
-      # &page={match}>
-      # &page={match}&...
-      # ```
-      page = Regex.run(~r{[&/?]page=([^>&]+)}, link) |> List.last |> String.to_integer
-
+      rel = get_rel(link)
+      page = get_page(link)
       {rel, page}
     end)
     |> Enum.into(%{})
     |> Map.get("last")
   end
+
+  defp get_rel(link) do
+    # Searches for `rel=`
+    Regex.run(~r{rel="([a-z]+)"}, link) |> List.last()
+  end
+
+  defp get_page(link) do
+    # Searches for the following variations:
+    # ```
+    # ?page={match}>
+    # ?page={match}&...
+    # &page={match}>
+    # &page={match}&...
+    # ```
+    Regex.run(~r{[&/?]page=([^>&]+)}, link) |> List.last |> String.to_integer
+  end
 end

lib/code_corps/github/api/repository.ex

@@ -12,8 +12,9 @@ defmodule CodeCorps.GitHub.API.Repository do
 
   @spec issues(GithubRepo.t) :: {:ok, list(map)} | {:error, GitHub.api_error_struct}
   def issues(%GithubRepo{github_app_installation: %GithubAppInstallation{} = installation} = github_repo) do
-    with {:ok, access_token} <- installation |> API.Installation.get_access_token(),
-      issues <- github_repo |> fetch_issues(access_token) do
+    with {:ok, access_token} <- API.Installation.get_access_token(installation),
+         issues <- fetch_issues(github_repo, access_token)
+    do
       {:ok, issues}
     else
       {:error, error} -> {:error, error}
@@ -25,24 +26,6 @@ defmodule CodeCorps.GitHub.API.Repository do
     path = "repos/#{owner}/#{repo}/issues"
     params = [per_page: per_page, state: "all"]
     opts = [access_token: access_token, params: params]
-
-    # stream/lazy
-    # before_operation = Timex.now
-    # results = path |> fetch_lazy(opts)
-    # after_operation = Timex.now
-    # count = results |> Enum.count
-    # pages = count / per_page |> Float.round
-    # elapsed = Timex.diff(after_operation, before_operation)
-    # IO.puts("Stream: #{count} records, #{per_page} records per page, #{pages} pages total, #{elapsed} microseconds")
-
-    # eager
-    GitHub.eager_get_all(path, %{}, opts)
-  end
-
-  defp fetch_lazy(path, opts) do
-    path
-    |> GitHub.lazy_get_all(%{}, opts)
-    |> Enum.to_list
-    |> List.flatten
+    GitHub.get_all(path, %{}, opts)
   end
 end

lib/code_corps/github/api/stream_api.ex

@@ -124,16 +124,8 @@ defmodule CodeCorps.GitHub do
     end
   end
 
-  def lazy_get_all(endpoint, headers, options) do
-    api().lazy_get_all(
-      api_url_for(endpoint),
-      headers |> Headers.user_request(options),
-      options |> add_default_options()
-    )
-  end
-
-  def eager_get_all(endpoint, headers, options) do
-    api().eager_get_all(
+  def get_all(endpoint, headers, options) do
+    api().get_all(
       api_url_for(endpoint),
       headers |> Headers.user_request(options),
       options |> add_default_options()