diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml index e4b5981..331b4e4 100644 --- a/.github/workflows/elixir.yml +++ b/.github/workflows/elixir.yml @@ -35,7 +35,7 @@ jobs: run: mix deps.get - name: Initialize services - run: FARSIDE_TEST=1 mix run update.exs + run: FARSIDE_TEST=1 mix run -e Farside.Instances.sync - name: Run tests - run: mix test --trace + run: FARSIDE_TEST=1 mix test --trace diff --git a/README.md b/README.md index a2dd1da..c00ac88 100644 --- a/README.md +++ b/README.md @@ -26,11 +26,11 @@ bottlenecks and rate-limiting. ## How It Works -The app runs in a container that periodically (default every 5 minutes) queries -all instances for services defined in [services.json](services.json). For each -instance, as long as the instance takes <5 seconds to respond and returns a 200 -status code, the instance is added to a list of available instances for that -particular service. If not, it is discarded until the next update period. +The app runs with an internally scheduled cron task that queries all instances +for services defined in [services.json](services.json) every 5 minutes. For +each instance, as long as the instance takes <5 seconds to respond and returns +a 200 status code, the instance is added to a list of available instances for +that particular service. If not, it is discarded until the next update period. Farside's routing is very minimal, with only the following routes: @@ -69,7 +69,7 @@ request per second per IP. - Install [elixir](https://elixir-lang.org/install.html) - Start redis: `redis-server /usr/local/etc/redis.conf` - Install dependencies: `mix deps.get` -- Initialize redis contents: `mix run update.exs` +- Initialize redis contents: `mix run -e Farside.Instances.sync` - Run Farside: `mix run --no-halt` - Uses localhost:4001 diff --git a/install-crontab.sh b/install-crontab.sh deleted file mode 100755 index 47c220c..0000000 --- a/install-crontab.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -# Install crontab to run update script - -SCRIPT_DIR="$(builtin cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -(crontab -l 2>/dev/null; echo "*/5 * * * * $SCRIPT_DIR/update.sh") | crontab - diff --git a/lib/farside/application.ex b/lib/farside/application.ex index d1c5e25..7427e89 100644 --- a/lib/farside/application.ex +++ b/lib/farside/application.ex @@ -7,7 +7,8 @@ defmodule Farside.Application do @impl true def start(_type, _args) do - plug_children = [ + + plug_children = System.get_env("FARSIDE_NO_ROUTER") && [] || [ Plug.Cowboy.child_spec( scheme: :http, plug: Farside.Router, @@ -19,9 +20,10 @@ defmodule Farside.Application do ] children = [ - {Redix, {@redis_conn, [name: :redix]}} | - System.get_env("FARSIDE_NO_ROUTER") && [] || plug_children - ] + {Redix, {@redis_conn, [name: :redix]}}, + Farside.Scheduler, + Farside.Server + ] ++ plug_children opts = [strategy: :one_for_one, name: Farside.Supervisor] Supervisor.start_link(children, opts) diff --git a/update.exs b/lib/farside/instances.ex similarity index 90% rename from update.exs rename to lib/farside/instances.ex index bc9ba0d..203ddfa 100644 --- a/update.exs +++ b/lib/farside/instances.ex @@ -1,12 +1,19 @@ -defmodule Instances do +defmodule Farside.Instances do @fallback_suffix Application.fetch_env!(:farside, :fallback_suffix) @update_file Application.fetch_env!(:farside, :update_file) @services_json Application.fetch_env!(:farside, :services_json) @service_prefix Application.fetch_env!(:farside, :service_prefix) - def init() do + def sync() do File.rename(@update_file, "#{@update_file}-prev") update() + + # Add UTC time of last update + Redix.command(:redix, [ + "SET", + "last_updated", + Calendar.strftime(DateTime.utc_now(), "%c") + ]) end def request(url) do @@ -24,7 +31,7 @@ defmodule Instances do end end - def update do + def update() do {:ok, file} = File.read(@services_json) {:ok, json} = Poison.decode(file, as: [%Service{}]) @@ -77,12 +84,3 @@ defmodule Instances do File.close(file) end end - -Instances.init() - -# Add UTC time of last update -Redix.command(:redix, [ - "SET", - "last_updated", - Calendar.strftime(DateTime.utc_now(), "%c") -]) diff --git a/lib/farside/scheduler.ex b/lib/farside/scheduler.ex new file mode 100644 index 0000000..4707624 --- /dev/null +++ b/lib/farside/scheduler.ex @@ -0,0 +1,3 @@ +defmodule Farside.Scheduler do + use Quantum, otp_app: :farside +end diff --git a/lib/farside/server.ex b/lib/farside/server.ex new file mode 100644 index 0000000..1d5bb76 --- /dev/null +++ b/lib/farside/server.ex @@ -0,0 +1,22 @@ +defmodule Farside.Server do + use GenServer + import Crontab.CronExpression + + def init(init_arg) do + {:ok, init_arg} + end + + def start_link(arg) do + if System.get_env("FARSIDE_TEST") do + IO.puts("Skipping sync job setup...") + else + Farside.Scheduler.new_job() + |> Quantum.Job.set_name(:sync) + |> Quantum.Job.set_schedule(~e[*/5 * * * *]) + |> Quantum.Job.set_task(fn -> Farside.Instances.sync end) + |> Farside.Scheduler.add_job() + end + + GenServer.start_link(__MODULE__, arg) + end +end diff --git a/mix.exs b/mix.exs index 6b9f512..229987e 100644 --- a/mix.exs +++ b/mix.exs @@ -27,6 +27,7 @@ defmodule Farside.MixProject do {:plug_attack, "~> 0.4.2"}, {:plug_cowboy, "~> 2.0"}, {:poison, "~> 5.0"}, + {:quantum, "~> 3.0"}, {:redix, "~> 1.1"} ] end diff --git a/mix.lock b/mix.lock index 26a5cc7..5f327f5 100644 --- a/mix.lock +++ b/mix.lock @@ -3,6 +3,8 @@ "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, + "crontab": {:hex, :crontab, "1.1.10", "dc9bb1f4299138d47bce38341f5dcbee0aa6c205e864fba7bc847f3b5cb48241", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "1347d889d1a0eda997990876b4894359e34bfbbd688acbb0ba28a2795ca40685"}, + "gen_stage": {:hex, :gen_stage, "1.1.2", "b1656cd4ba431ed02c5656fe10cb5423820847113a07218da68eae5d6a260c23", [:mix], [], "hexpm", "9e39af23140f704e2b07a3e29d8f05fd21c2aaf4088ff43cb82be4b9e3148d02"}, "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"}, "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, @@ -18,6 +20,7 @@ "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"}, "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"}, "poison": {:hex, :poison, "5.0.0", "d2b54589ab4157bbb82ec2050757779bfed724463a544b6e20d79855a9e43b24", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "11dc6117c501b80c62a7594f941d043982a1bd05a1184280c0d9166eb4d8d3fc"}, + "quantum": {:hex, :quantum, "3.4.0", "5a53c3c52b0d55f2323940232ba6ab4c98e7e14c73dfacbba3a1ed799b037ce5", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d0eb64957d3dc49c8ed730cc2203108334226496535965b8dfa3f3dbcf430f87"}, "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "redix": {:hex, :redix, "1.1.4", "d66fc83d2d4f136c838568d1ec8b0c1a72acfcecfac88a40f86f60aaee883c93", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "515eff055b7de8967e835f4de22a6cfe8311bc1b8fe72f48200238fb43f6a803"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"}, diff --git a/update.sh b/update.sh deleted file mode 100755 index ff14797..0000000 --- a/update.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh - -SCRIPT_DIR="$(builtin cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)" - -cd "$SCRIPT_DIR" -FARSIDE_NO_ROUTER=1 mix run update.exs