Use quantum core for update scheduling

Rather than requiring a traditional crontab install, the app now
leverages quantum-core (link below) to schedule the instance update/sync
task every 5 minutes. Some updates as a result:

  - The new job is scheduled at runtime in server.ex.
  - The update.exs script was refactored to be compiled along with the
    rest of the app as instances.ex.
  - Scheduler and Server modules were added for creating and executing
    the new update task
  - All shell scripts were removed, as they are no longer needed

https://github.com/quantum-elixir/quantum-core

.github/workflows/elixir.yml

@@ -35,7 +35,7 @@ jobs:
       run: mix deps.get
 
     - name: Initialize services
-      run: FARSIDE_TEST=1 mix run update.exs
+      run: FARSIDE_TEST=1 mix run -e Farside.Instances.sync
 
     - name: Run tests
-      run: mix test --trace
+      run: FARSIDE_TEST=1 mix test --trace

README.md

@@ -26,11 +26,11 @@ bottlenecks and rate-limiting.
 
 ## How It Works
 
-The app runs in a container that periodically (default every 5 minutes) queries
+The app runs with an internally scheduled cron task that queries all instances
-all instances for services defined in [services.json](services.json). For each
+for services defined in [services.json](services.json) every 5 minutes. For
-instance, as long as the instance takes <5 seconds to respond and returns a 200
+each instance, as long as the instance takes <5 seconds to respond and returns
-status code, the instance is added to a list of available instances for that
+a 200 status code, the instance is added to a list of available instances for
-particular service. If not, it is discarded until the next update period. 
+that particular service. If not, it is discarded until the next update period.
 
 Farside's routing is very minimal, with only the following routes:
 
@@ -69,7 +69,7 @@ request per second per IP.
 - Install [elixir](https://elixir-lang.org/install.html)
 - Start redis: `redis-server /usr/local/etc/redis.conf`
 - Install dependencies: `mix deps.get`
-- Initialize redis contents: `mix run update.exs`
+- Initialize redis contents: `mix run -e Farside.Instances.sync`
 - Run Farside: `mix run --no-halt`
   - Uses localhost:4001
 

install-crontab.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-# Install crontab to run update script
-
-SCRIPT_DIR="$(builtin cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
-
-(crontab -l 2>/dev/null; echo "*/5 * * * * $SCRIPT_DIR/update.sh") | crontab -

lib/farside/application.ex

@@ -7,7 +7,8 @@ defmodule Farside.Application do
 
   @impl true
   def start(_type, _args) do
-    plug_children = [
+
+    plug_children = System.get_env("FARSIDE_NO_ROUTER") && [] || [
       Plug.Cowboy.child_spec(
         scheme: :http,
         plug: Farside.Router,
@@ -19,9 +20,10 @@ defmodule Farside.Application do
     ]
 
     children = [
-      {Redix, {@redis_conn, [name: :redix]}} |
+      {Redix, {@redis_conn, [name: :redix]}},
-      System.get_env("FARSIDE_NO_ROUTER") && [] || plug_children
+      Farside.Scheduler,
-    ]
+      Farside.Server
+    ] ++ plug_children
 
     opts = [strategy: :one_for_one, name: Farside.Supervisor]
     Supervisor.start_link(children, opts)

lib/farside/instances.ex

@@ -1,12 +1,19 @@
-defmodule Instances do
+defmodule Farside.Instances do
   @fallback_suffix Application.fetch_env!(:farside, :fallback_suffix)
   @update_file Application.fetch_env!(:farside, :update_file)
   @services_json Application.fetch_env!(:farside, :services_json)
   @service_prefix Application.fetch_env!(:farside, :service_prefix)
 
-  def init() do
+  def sync() do
     File.rename(@update_file, "#{@update_file}-prev")
     update()
+
+    # Add UTC time of last update
+    Redix.command(:redix, [
+      "SET",
+      "last_updated",
+      Calendar.strftime(DateTime.utc_now(), "%c")
+    ])
   end
 
   def request(url) do
@@ -24,7 +31,7 @@ defmodule Instances do
     end
   end
 
-  def update do
+  def update() do
     {:ok, file} = File.read(@services_json)
     {:ok, json} = Poison.decode(file, as: [%Service{}])
 
@@ -77,12 +84,3 @@ defmodule Instances do
     File.close(file)
   end
 end
-
-Instances.init()
-
-# Add UTC time of last update
-Redix.command(:redix, [
-  "SET",
-  "last_updated",
-  Calendar.strftime(DateTime.utc_now(), "%c")
-])

lib/farside/scheduler.ex

@@ -0,0 +1,3 @@
+defmodule Farside.Scheduler do
+  use Quantum, otp_app: :farside
+end

lib/farside/server.ex

@@ -0,0 +1,22 @@
+defmodule Farside.Server do
+  use GenServer
+  import Crontab.CronExpression
+
+  def init(init_arg) do
+    {:ok, init_arg}
+  end
+
+  def start_link(arg) do
+    if System.get_env("FARSIDE_TEST") do
+      IO.puts("Skipping sync job setup...")
+    else
+      Farside.Scheduler.new_job()
+      |> Quantum.Job.set_name(:sync)
+      |> Quantum.Job.set_schedule(~e[*/5 * * * *])
+      |> Quantum.Job.set_task(fn -> Farside.Instances.sync end)
+      |> Farside.Scheduler.add_job()
+    end
+
+    GenServer.start_link(__MODULE__, arg)
+  end
+end

mix.exs

@@ -27,6 +27,7 @@ defmodule Farside.MixProject do
       {:plug_attack, "~> 0.4.2"},
       {:plug_cowboy, "~> 2.0"},
       {:poison, "~> 5.0"},
+      {:quantum, "~> 3.0"},
       {:redix, "~> 1.1"}
     ]
   end

mix.lock

@@ -3,6 +3,8 @@
   "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"},
   "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"},
   "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"},
+  "crontab": {:hex, :crontab, "1.1.10", "dc9bb1f4299138d47bce38341f5dcbee0aa6c205e864fba7bc847f3b5cb48241", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "1347d889d1a0eda997990876b4894359e34bfbbd688acbb0ba28a2795ca40685"},
+  "gen_stage": {:hex, :gen_stage, "1.1.2", "b1656cd4ba431ed02c5656fe10cb5423820847113a07218da68eae5d6a260c23", [:mix], [], "hexpm", "9e39af23140f704e2b07a3e29d8f05fd21c2aaf4088ff43cb82be4b9e3148d02"},
   "hackney": {:hex, :hackney, "1.18.0", "c4443d960bb9fba6d01161d01cd81173089686717d9490e5d3606644c48d121f", [:rebar3], [{:certifi, "~>2.8.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "9afcda620704d720db8c6a3123e9848d09c87586dc1c10479c42627b905b5c5e"},
   "httpoison": {:hex, :httpoison, "1.8.0", "6b85dea15820b7804ef607ff78406ab449dd78bed923a49c7160e1886e987a3d", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "28089eaa98cf90c66265b6b5ad87c59a3729bea2e74e9d08f9b51eb9729b3c3a"},
   "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"},
@@ -18,6 +20,7 @@
   "plug_cowboy": {:hex, :plug_cowboy, "2.5.2", "62894ccd601cf9597e2c23911ff12798a8a18d237e9739f58a6b04e4988899fe", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "ea6e87f774c8608d60c8d34022a7d073bd7680a0a013f049fc62bf35efea1044"},
   "plug_crypto": {:hex, :plug_crypto, "1.2.2", "05654514ac717ff3a1843204b424477d9e60c143406aa94daf2274fdd280794d", [:mix], [], "hexpm", "87631c7ad914a5a445f0a3809f99b079113ae4ed4b867348dd9eec288cecb6db"},
   "poison": {:hex, :poison, "5.0.0", "d2b54589ab4157bbb82ec2050757779bfed724463a544b6e20d79855a9e43b24", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "11dc6117c501b80c62a7594f941d043982a1bd05a1184280c0d9166eb4d8d3fc"},
+  "quantum": {:hex, :quantum, "3.4.0", "5a53c3c52b0d55f2323940232ba6ab4c98e7e14c73dfacbba3a1ed799b037ce5", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "d0eb64957d3dc49c8ed730cc2203108334226496535965b8dfa3f3dbcf430f87"},
   "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"},
   "redix": {:hex, :redix, "1.1.4", "d66fc83d2d4f136c838568d1ec8b0c1a72acfcecfac88a40f86f60aaee883c93", [:mix], [{:castore, "~> 0.1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "515eff055b7de8967e835f4de22a6cfe8311bc1b8fe72f48200238fb43f6a803"},
   "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.6", "cf344f5692c82d2cd7554f5ec8fd961548d4fd09e7d22f5b62482e5aeaebd4b0", [:make, :mix, :rebar3], [], "hexpm", "bdb0d2471f453c88ff3908e7686f86f9be327d065cc1ec16fa4540197ea04680"},

update.sh

@@ -1,6 +0,0 @@
-#!/bin/sh
-
-SCRIPT_DIR="$(builtin cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
-
-cd "$SCRIPT_DIR"
-FARSIDE_NO_ROUTER=1 mix run update.exs