Skip to content

Commit dc807dd

Browse files
committed
chore: speed up engine builds with incremental namespacing
1 parent 3d0dfb5 commit dc807dd

File tree

9 files changed

+207
-21
lines changed

9 files changed

+207
-21
lines changed

apps/engine/mix.lock

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
%{
22
"benchee": {:hex, :benchee, "1.3.1", "c786e6a76321121a44229dde3988fc772bca73ea75170a73fd5f4ddf1af95ccf", [:mix], [{:deep_merge, "~> 1.0", [hex: :deep_merge, repo: "hexpm", optional: false]}, {:statistex, "~> 1.0", [hex: :statistex, repo: "hexpm", optional: false]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "76224c58ea1d0391c8309a8ecbfe27d71062878f59bd41a390266bf4ac1cc56d"},
3+
"briefly": {:hex, :briefly, "0.5.1", "ee10d48da7f79ed2aebdc3e536d5f9a0c3e36ff76c0ad0d4254653a152b13a8a", [:mix], [], "hexpm", "bd684aa92ad8b7b4e0d92c31200993c4bc1469fc68cd6d5f15144041bd15cb57"},
34
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
45
"castore": {:hex, :castore, "1.0.12", "053f0e32700cbec356280c0e835df425a3be4bc1e0627b714330ad9d0f05497f", [:mix], [], "hexpm", "3dca286b2186055ba0c9449b4e95b97bf1b57b47c1f2644555879e659960c224"},
56
"credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"},

apps/expert/lib/expert/engine_node.ex

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ defmodule Expert.EngineNode do
8888
IO.puts("ok")
8989

9090
{:error, reason} ->
91-
IO.puts("error starting node:\n \#{inspect(reason)}")
91+
IO.puts("error starting node:\n #{inspect(reason)}")
9292
end
9393
end
9494

apps/expert/lib/expert/release.ex

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
defmodule Expert.Release do
22
def assemble(release) do
3-
Mix.Task.run(:namespace, [release.path])
3+
# In-place namespacing: both source and output are the same path
4+
Mix.Task.run(:namespace, [release.path, release.path])
45

56
expert_root = Path.expand("../../../..", __DIR__)
67
engine_path = Path.join([expert_root, "apps", "engine"])

apps/expert/mix.lock

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
%{
2+
"briefly": {:hex, :briefly, "0.5.1", "ee10d48da7f79ed2aebdc3e536d5f9a0c3e36ff76c0ad0d4254653a152b13a8a", [:mix], [], "hexpm", "bd684aa92ad8b7b4e0d92c31200993c4bc1469fc68cd6d5f15144041bd15cb57"},
23
"bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"},
34
"burrito": {:hex, :burrito, "1.5.0", "d68ec01df2871f1d5bc603b883a78546c75761ac73c1bec1b7ae2cc74790fcd1", [:mix], [{:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}, {:req, ">= 0.5.0", [hex: :req, repo: "hexpm", optional: false]}, {:typed_struct, "~> 0.2.0 or ~> 0.3.0", [hex: :typed_struct, repo: "hexpm", optional: false]}], "hexpm", "3861abda7bffa733862b48da3e03df0b4cd41abf6fd24b91745f5c16d971e5fa"},
45
"credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"},

apps/expert/priv/build_engine.exs

Lines changed: 18 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,23 +17,29 @@ expert_data_path = :filename.basedir(:user_data, "Expert", %{version: expert_vsn
1717

1818
System.put_env("MIX_INSTALL_DIR", expert_data_path)
1919

20-
Mix.Task.run("local.hex", ["--force"])
21-
Mix.Task.run("local.rebar", ["--force"])
22-
23-
Mix.install([{:engine, path: engine_source_path, env: :dev}],
24-
start_applications: false,
25-
config_path: Path.join(engine_source_path, "config/config.exs"),
26-
lockfile: Path.join(engine_source_path, "mix.lock")
27-
)
20+
compile_engine =
21+
fn ->
22+
Mix.install([{:engine, path: engine_source_path, env: :dev}],
23+
start_applications: false,
24+
config_path: Path.join(engine_source_path, "config/config.exs"),
25+
lockfile: Path.join(engine_source_path, "mix.lock")
26+
)
27+
end
28+
29+
try do
30+
compile_engine.()
31+
rescue
32+
_e ->
33+
Mix.Task.run("local.hex", ["--force"])
34+
Mix.Task.run("local.rebar", ["--force"])
35+
compile_engine.()
36+
end
2837

2938
install_path = Mix.install_project_dir()
3039

3140
dev_build_path = Path.join([install_path, "_build", "dev"])
3241
ns_build_path = Path.join([install_path, "_build", "dev_ns"])
3342

34-
File.rm_rf!(ns_build_path)
35-
File.cp_r!(dev_build_path, ns_build_path)
36-
37-
Mix.Task.run("namespace", [ns_build_path, "--cwd", install_path])
43+
Mix.Task.run("namespace", [dev_build_path, ns_build_path, "--cwd", install_path])
3844

3945
IO.puts("engine_path:" <> ns_build_path)
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
defmodule Forge.Namespace.FileSync do
2+
3+
defmodule Classification do
4+
defstruct changed: [],
5+
new: [],
6+
deleted: []
7+
end
8+
9+
alias __MODULE__.Classification
10+
11+
@doc """
12+
Classifies files into changed, new, and deleted categories.
13+
14+
It looks at `**/ebin/*` files in both the base directory and output directory,
15+
applying namespacing to the file names in the output directory.
16+
Files can be `.beam` or `.app` files.
17+
18+
Then compares the mtimes of the files to determine their classification.
19+
20+
If files in output_directory are not present in base_directory, they are classified as deleted.
21+
"""
22+
def classify_files(same, same), do: %Classification{
23+
changed: [],
24+
new: [],
25+
deleted: []
26+
}
27+
28+
def classify_files(base_directory, output_directory) do
29+
# Files in base directory are not namespaced, eg:
30+
# lib/foo/ebin/Elixir.Foo.Bar.beam
31+
# lib/foo/ebin/foo.app
32+
#
33+
# Files in output directory are namespaced, eg:
34+
# lib/xp_foo/ebin/Elixir.XPFoo.Bar.beam
35+
# lib/xp_foo/ebin/xp_foo.app
36+
#
37+
# We need to compare the files by applying namespacing to the base directory paths,
38+
# then comparing mtimes.
39+
#
40+
# For any files in output_directory that don't have a corresponding file in base_directory,
41+
# we classify them as deleted.
42+
43+
base_files = find_files(Path.join(base_directory, "lib"))
44+
output_files = find_files(Path.join(output_directory, "lib"))
45+
46+
base_map =
47+
Enum.reduce(base_files, %{}, fn base_file, acc ->
48+
relative_path = Path.relative_to(base_file, base_directory)
49+
namespaced_relative_path =
50+
relative_path
51+
|> Forge.Namespace.Path.apply()
52+
|> maybe_namespace_filename()
53+
54+
dest_path = Path.join(output_directory, namespaced_relative_path)
55+
Map.put(acc, base_file, dest_path)
56+
end)
57+
expected_dest_files = Map.values(base_map) |> MapSet.new()
58+
output_set = MapSet.new(output_files)
59+
60+
classification =
61+
Enum.reduce(base_map, %Classification{}, fn {base_file, dest_path}, acc ->
62+
if File.exists?(dest_path) do
63+
base_mtime = File.stat!(base_file).mtime
64+
output_mtime = File.stat!(dest_path).mtime
65+
66+
if base_mtime > output_mtime do
67+
%{acc | changed: [{base_file, dest_path} | acc.changed]}
68+
else
69+
acc
70+
end
71+
else
72+
%{acc | new: [{base_file, dest_path} | acc.new]}
73+
end
74+
end)
75+
76+
deleted_files =
77+
output_set
78+
|> MapSet.difference(MapSet.new(expected_dest_files))
79+
|> MapSet.to_list()
80+
81+
%{classification | deleted: deleted_files}
82+
end
83+
84+
defp find_files(directory) do
85+
[directory, "**", "*"]
86+
|> Path.join()
87+
|> Path.wildcard()
88+
|> Enum.filter(&File.regular?/1)
89+
end
90+
91+
defp maybe_namespace_filename(file_path) do
92+
# namespace filename for .beam and .app files
93+
extname = Path.extname(file_path)
94+
95+
if extname in [".beam", ".app"] do
96+
dirname = Path.dirname(file_path)
97+
basename = Path.basename(file_path, extname)
98+
namespaced_basename = Forge.Namespace.Module.apply(String.to_atom(basename)) |> Atom.to_string()
99+
Path.join(dirname, namespaced_basename <> extname)
100+
else
101+
file_path
102+
end
103+
end
104+
end
Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
defmodule Mix.Tasks.Classify do
2+
use Mix.Task
3+
4+
def run([base_directory, output_directory]) do
5+
classified_files = Forge.Namespace.FileSync.classify_files(base_directory, output_directory)
6+
7+
Mix.Shell.IO.info("Changed files:")
8+
Enum.each(classified_files.changed, fn {base, output} ->
9+
Mix.Shell.IO.info(" Changed: #{base} -> #{output}")
10+
end)
11+
12+
Mix.Shell.IO.info("New files:")
13+
Enum.each(classified_files.new, fn {base, output} ->
14+
Mix.Shell.IO.info(" New: #{base} -> #{output}")
15+
end)
16+
17+
Mix.Shell.IO.info("Deleted files:")
18+
Enum.each(classified_files.deleted, fn output ->
19+
Mix.Shell.IO.info(" Deleted: #{output}")
20+
end)
21+
end
22+
end

apps/forge/lib/mix/tasks/namespace.ex

Lines changed: 57 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,7 @@ defmodule Mix.Tasks.Namespace do
3030

3131
require Logger
3232

33-
def run([base_directory | opts]) do
33+
def run([base_directory, output_directory | opts]) do
3434
{args, _, _} =
3535
OptionParser.parse(opts,
3636
strict: [cwd: :string]
@@ -44,14 +44,64 @@ defmodule Mix.Tasks.Namespace do
4444
# Otherwise only the @extra_apps will be cached
4545
init()
4646

47-
Transform.Apps.apply_to_all(base_directory)
48-
Transform.Beams.apply_to_all(base_directory)
49-
Transform.Scripts.apply_to_all(base_directory)
47+
File.mkdir_p!(output_directory)
48+
49+
if base_directory == output_directory do
50+
apply_transforms(base_directory)
51+
else
52+
incremental_transforms(base_directory, output_directory)
53+
end
54+
end
55+
56+
defp apply_transforms(directory) do
57+
Transform.Apps.apply_to_all(directory)
58+
Transform.Beams.apply_to_all(directory)
59+
Transform.Scripts.apply_to_all(directory)
5060
# The boot file transform just turns script files into boot files
5161
# so it must come after the script file transform
52-
Transform.Boots.apply_to_all(base_directory)
53-
Transform.Configs.apply_to_all(base_directory)
54-
Transform.AppDirectories.apply_to_all(base_directory)
62+
Transform.Boots.apply_to_all(directory)
63+
Transform.Configs.apply_to_all(directory)
64+
Transform.AppDirectories.apply_to_all(directory)
65+
end
66+
67+
defp incremental_transforms(base_directory, output_directory) do
68+
Application.ensure_all_started(:briefly)
69+
70+
classification =
71+
Forge.Namespace.FileSync.classify_files(base_directory, output_directory)
72+
73+
tmp_dir = Briefly.create!(directory: true)
74+
75+
entries_to_namespace =
76+
classification.new ++ classification.changed
77+
78+
Mix.Shell.IO.info("""
79+
Namespacing #{length(entries_to_namespace)} files:
80+
New: #{length(classification.new)}
81+
Changed: #{length(classification.changed)}
82+
Deleted: #{length(classification.deleted)}
83+
""")
84+
85+
# Copy new and changed files to a temp directory
86+
Enum.each(entries_to_namespace, fn {src, _dest} ->
87+
relative_path = Path.relative_to(src, base_directory)
88+
tmp_dest = Path.join(tmp_dir, relative_path)
89+
File.mkdir_p!(Path.dirname(tmp_dest))
90+
File.cp!(src, tmp_dest)
91+
end)
92+
93+
# Delete removed files from output directory
94+
Enum.each(classification.deleted, fn dest ->
95+
if File.exists?(dest) do
96+
File.rm!(dest)
97+
end
98+
end)
99+
100+
# Apply transforms to temp directory
101+
apply_transforms(tmp_dir)
102+
103+
# Copy temp directory back to output directory
104+
File.cp_r!(tmp_dir, output_directory)
55105
end
56106

57107
def app_names do

apps/forge/mix.exs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ defmodule Forge.MixProject do
3636
defp deps do
3737
[
3838
{:benchee, "~> 1.3", only: :test},
39+
{:briefly, "~> 0.5"},
3940
Mix.Credo.dependency(),
4041
Mix.Dialyzer.dependency(),
4142
{:deps_nix, "~> 2.4", only: :dev},

0 commit comments

Comments
 (0)