mirror of
https://github.com/benbusby/farside.git
synced 2025-06-08 10:26:36 +00:00
refactor
This commit is contained in:
parent
03e46f81f8
commit
84caea3af2
17 changed files with 402 additions and 140 deletions
|
@ -78,7 +78,18 @@ defmodule Farside.Application do
|
|||
|
||||
struct(%Service{}, service_atom)
|
||||
|> Farside.Instance.Supervisor.start()
|
||||
|> HealthyCheck.load()
|
||||
end
|
||||
|
||||
response
|
||||
|> maybe_run()
|
||||
end
|
||||
|
||||
def maybe_run(response) do
|
||||
if is_nil(System.get_env("FARSIDE_TEST")) do
|
||||
Task.start(fn ->
|
||||
Process.sleep(10_000)
|
||||
UnHealthyCheck.run()
|
||||
end)
|
||||
end
|
||||
|
||||
response
|
||||
|
|
|
@ -5,6 +5,8 @@ defmodule Farside.Server.DeadCheck do
|
|||
use Task
|
||||
alias Farside.LastUpdated
|
||||
|
||||
require Logger
|
||||
|
||||
def child_spec(args) do
|
||||
%{
|
||||
id: __MODULE__,
|
||||
|
@ -20,7 +22,7 @@ defmodule Farside.Server.DeadCheck do
|
|||
def poll() do
|
||||
receive do
|
||||
after
|
||||
1_200_000 ->
|
||||
86_400_000 ->
|
||||
run()
|
||||
poll()
|
||||
end
|
||||
|
@ -29,6 +31,8 @@ defmodule Farside.Server.DeadCheck do
|
|||
def run() do
|
||||
LastUpdated.value(DateTime.utc_now())
|
||||
|
||||
Logger.info("Dead Service Check Running")
|
||||
|
||||
Registry.dispatch(:status, "dead", fn entries ->
|
||||
for {pid, _} <- entries, do: GenServer.cast(pid, :check)
|
||||
end)
|
||||
|
|
|
@ -5,6 +5,8 @@ defmodule Farside.Server.HealthyCheck do
|
|||
use Task
|
||||
alias Farside.LastUpdated
|
||||
|
||||
require Logger
|
||||
|
||||
def child_spec(args) do
|
||||
%{
|
||||
id: __MODULE__,
|
||||
|
@ -20,27 +22,19 @@ defmodule Farside.Server.HealthyCheck do
|
|||
def poll() do
|
||||
receive do
|
||||
after
|
||||
90_000 ->
|
||||
300_000 ->
|
||||
run()
|
||||
poll()
|
||||
end
|
||||
end
|
||||
|
||||
def load(params) do
|
||||
Registry.dispatch(:status, "healthy", fn entries ->
|
||||
for {pid, url} <- entries do
|
||||
GenServer.cast(pid, :check)
|
||||
end
|
||||
end)
|
||||
|
||||
params
|
||||
end
|
||||
|
||||
def run() do
|
||||
LastUpdated.value(DateTime.utc_now())
|
||||
|
||||
Logger.info("Healthy Service Check Running")
|
||||
|
||||
Registry.dispatch(:status, "healthy", fn entries ->
|
||||
for {pid, url} <- entries do
|
||||
for {pid, _url} <- entries do
|
||||
GenServer.cast(pid, :check)
|
||||
end
|
||||
end)
|
||||
|
|
|
@ -1,6 +1,11 @@
|
|||
defmodule Farside.Http do
|
||||
require Logger
|
||||
|
||||
@moduledoc """
|
||||
Http
|
||||
the http client
|
||||
"""
|
||||
|
||||
@headers Application.fetch_env!(:farside, :headers)
|
||||
@queries Application.fetch_env!(:farside, :queries)
|
||||
@recv_timeout String.to_integer(Application.fetch_env!(:farside, :recv_timeout))
|
||||
|
@ -108,11 +113,11 @@ defmodule Farside.Http do
|
|||
nil
|
||||
end
|
||||
|
||||
unless is_nil(data) do
|
||||
if is_nil(data) do
|
||||
:bad
|
||||
else
|
||||
{_test_url, value, _service} = data
|
||||
value
|
||||
else
|
||||
:bad
|
||||
end
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,9 +1,12 @@
|
|||
defmodule Farside.Instance do
|
||||
use GenServer
|
||||
|
||||
require Logger
|
||||
@moduledoc """
|
||||
Instance
|
||||
this will store the pointer to ets
|
||||
"""
|
||||
|
||||
alias Farside.Http
|
||||
require Logger
|
||||
|
||||
@registry_name :instance
|
||||
|
||||
|
@ -15,6 +18,7 @@ defmodule Farside.Instance do
|
|||
}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(init_arg) do
|
||||
ref =
|
||||
:ets.new(String.to_atom(init_arg.type), [
|
||||
|
@ -40,6 +44,7 @@ defmodule Farside.Instance do
|
|||
GenServer.call(__MODULE__, :shutdown)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(
|
||||
:shutdown,
|
||||
_from,
|
||||
|
@ -48,6 +53,7 @@ defmodule Farside.Instance do
|
|||
{:stop, {:ok, "Normal Shutdown"}, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast(
|
||||
:shutdown,
|
||||
state
|
||||
|
@ -61,7 +67,7 @@ defmodule Farside.Instance do
|
|||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:DOWN, ref, :process, _pid, _reason}, {names, refs}) do
|
||||
def handle_info({:DOWN, _ref, :process, _pid, _reason}, {names, refs}) do
|
||||
:ets.delete(names)
|
||||
{:noreply, {names, refs}}
|
||||
end
|
||||
|
|
|
@ -1,7 +1,11 @@
|
|||
defmodule Farside.Instance.Supervisor do
|
||||
use DynamicSupervisor
|
||||
|
||||
alias __MODULE__, as: SUPERVISOR
|
||||
@moduledoc """
|
||||
Instance Supervisor
|
||||
this will supervise the instance
|
||||
"""
|
||||
|
||||
alias Farside.Instance, as: SERVER
|
||||
|
||||
@name :instance_supervisor
|
||||
|
|
|
@ -1,6 +1,8 @@
|
|||
defmodule Farside.LastUpdated do
|
||||
use Agent
|
||||
|
||||
@moduledoc nil
|
||||
|
||||
def start_link(initial_value) do
|
||||
Agent.start_link(fn -> initial_value end, name: __MODULE__)
|
||||
end
|
||||
|
|
|
@ -1,6 +1,11 @@
|
|||
defmodule Farside.Service do
|
||||
use GenServer
|
||||
|
||||
@moduledoc """
|
||||
Service
|
||||
this will store the service state
|
||||
"""
|
||||
|
||||
require Logger
|
||||
|
||||
alias Farside.Http
|
||||
|
@ -21,19 +26,22 @@ defmodule Farside.Service do
|
|||
}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def init(data) do
|
||||
initial_state = %__MODULE__{
|
||||
url: data.url,
|
||||
type: data.type,
|
||||
test_url: data.test_url,
|
||||
last_update: nil,
|
||||
last_update:
|
||||
DateTime.utc_now()
|
||||
|> DateTime.add(-86_400, :second),
|
||||
status: []
|
||||
}
|
||||
|
||||
healthy = "#{data.type}_healthy"
|
||||
unhealthy = "#{data.type}_unhealthy"
|
||||
|
||||
Registry.register(:status, healthy, data.url)
|
||||
Registry.register(:status, "healthy", data.url)
|
||||
Registry.register(:status, unhealthy, data.url)
|
||||
Registry.register(:status, "unhealthy", data.url)
|
||||
{:ok, initial_state}
|
||||
end
|
||||
|
||||
|
@ -46,6 +54,7 @@ defmodule Farside.Service do
|
|||
GenServer.call(__MODULE__, :shutdown)
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_call(
|
||||
:shutdown,
|
||||
_from,
|
||||
|
@ -54,28 +63,6 @@ defmodule Farside.Service do
|
|||
{:stop, {:ok, "Normal Shutdown"}, state}
|
||||
end
|
||||
|
||||
def handle_cast(
|
||||
:shutdown,
|
||||
state
|
||||
) do
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
@doc false
|
||||
def via_tuple(id, registry \\ @registry_name) do
|
||||
{:via, Registry, {registry, id}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:DOWN, ref, :process, _pid, _reason}, data) do
|
||||
{:noreply, data}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(_msg, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast(:load, state) do
|
||||
reply = Http.test_service(state)
|
||||
|
@ -91,57 +78,90 @@ defmodule Farside.Service do
|
|||
|
||||
@impl true
|
||||
def handle_cast(:check, state) do
|
||||
reply = Http.test_service(state)
|
||||
|
||||
status = state.status ++ [reply]
|
||||
|
||||
max_queue = Application.get_env(:farside, :max_fail_rate, 50) + 5
|
||||
|
||||
status =
|
||||
case Enum.count(status) < max_queue do
|
||||
true -> status
|
||||
false -> []
|
||||
end
|
||||
|
||||
state = %{state | status: status}
|
||||
|
||||
state = %{state | last_update: DateTime.utc_now()}
|
||||
|
||||
healthy = "#{state.type}_healthy"
|
||||
unhealthy = "#{state.type}_unhealthy"
|
||||
dead = "#{state.type}_dead"
|
||||
|
||||
Registry.unregister_match(:status, "healthy", state.url)
|
||||
Registry.unregister_match(:status, "unhealthy", state.url)
|
||||
Registry.unregister_match(:status, "dead", state.url)
|
||||
|
||||
Registry.unregister_match(:status, healthy, state.url)
|
||||
Registry.unregister_match(:status, unhealthy, state.url)
|
||||
Registry.unregister_match(:status, dead, state.url)
|
||||
dt =
|
||||
DateTime.utc_now()
|
||||
|> DateTime.add(-60, :second)
|
||||
|
||||
state =
|
||||
if reply != :good do
|
||||
filtered = Enum.reject(status, fn x -> x == :good end)
|
||||
case DateTime.compare(dt, state.last_update) do
|
||||
:gt ->
|
||||
reply = Http.test_service(state)
|
||||
|
||||
fails_before_death = Application.get_env(:farside, :max_fail_rate, 50)
|
||||
status = state.status ++ [reply]
|
||||
|
||||
case Enum.count(filtered) < fails_before_death do
|
||||
true ->
|
||||
Registry.register(:status, "unhealthy", state.url)
|
||||
Registry.register(:status, unhealthy, state.url)
|
||||
max_queue = Application.get_env(:farside, :max_fail_rate, 50) + 5
|
||||
|
||||
status =
|
||||
case Enum.count(status) < max_queue do
|
||||
true -> status
|
||||
false -> []
|
||||
end
|
||||
|
||||
state = %{state | status: status}
|
||||
|
||||
state = %{state | last_update: DateTime.utc_now()}
|
||||
|
||||
healthy = "#{state.type}_healthy"
|
||||
unhealthy = "#{state.type}_unhealthy"
|
||||
dead = "#{state.type}_dead"
|
||||
|
||||
Registry.unregister_match(:status, "healthy", state.url)
|
||||
Registry.unregister_match(:status, "unhealthy", state.url)
|
||||
Registry.unregister_match(:status, "dead", state.url)
|
||||
|
||||
Registry.unregister_match(:status, healthy, state.url)
|
||||
Registry.unregister_match(:status, unhealthy, state.url)
|
||||
Registry.unregister_match(:status, dead, state.url)
|
||||
|
||||
if reply != :good do
|
||||
filtered = Enum.reject(status, fn x -> x == :good end)
|
||||
|
||||
fails_before_death = Application.get_env(:farside, :max_fail_rate, 50)
|
||||
|
||||
case Enum.count(filtered) < fails_before_death do
|
||||
true ->
|
||||
Registry.register(:status, "unhealthy", state.url)
|
||||
Registry.register(:status, unhealthy, state.url)
|
||||
state
|
||||
|
||||
false ->
|
||||
Registry.register(:status, "dead", state.url)
|
||||
Registry.register(:status, dead, state.url)
|
||||
%{state | status: [:bad]}
|
||||
end
|
||||
else
|
||||
Registry.register(:status, "healthy", state.url)
|
||||
Registry.register(:status, healthy, state.url)
|
||||
state
|
||||
end
|
||||
|
||||
false ->
|
||||
Registry.register(:status, "dead", state.url)
|
||||
Registry.register(:status, dead, state.url)
|
||||
%{state | status: [:bad]}
|
||||
end
|
||||
else
|
||||
Registry.register(:status, "healthy", state.url)
|
||||
Registry.register(:status, healthy, state.url)
|
||||
state
|
||||
_ ->
|
||||
%{state | last_update: DateTime.utc_now()}
|
||||
end
|
||||
|
||||
{:noreply, state}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_cast(
|
||||
:shutdown,
|
||||
state
|
||||
) do
|
||||
{:stop, :normal, state}
|
||||
end
|
||||
|
||||
@doc false
|
||||
def via_tuple(id, registry \\ @registry_name) do
|
||||
{:via, Registry, {registry, id}}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info({:DOWN, _ref, :process, _pid, _reason}, data) do
|
||||
{:noreply, data}
|
||||
end
|
||||
|
||||
@impl true
|
||||
def handle_info(_msg, state) do
|
||||
{:noreply, state}
|
||||
end
|
||||
end
|
||||
|
|
|
@ -1,6 +1,11 @@
|
|||
defmodule Farside.Service.Supervisor do
|
||||
use DynamicSupervisor
|
||||
|
||||
@moduledoc """
|
||||
Service Supervisor
|
||||
this will supervise the service
|
||||
"""
|
||||
|
||||
alias __MODULE__, as: SUPERVISOR
|
||||
alias Farside.Service, as: SERVER
|
||||
|
||||
|
|
|
@ -2,6 +2,8 @@ defmodule Farside.Throttle do
|
|||
import Plug.Conn
|
||||
use PlugAttack
|
||||
|
||||
@moduledoc nil
|
||||
|
||||
rule "throttle per ip", conn do
|
||||
# throttle to 1 request per second
|
||||
throttle(conn.remote_ip,
|
||||
|
|
|
@ -5,6 +5,8 @@ defmodule Farside.Server.UnHealthyCheck do
|
|||
use Task
|
||||
alias Farside.LastUpdated
|
||||
|
||||
require Logger
|
||||
|
||||
def child_spec(args) do
|
||||
%{
|
||||
id: __MODULE__,
|
||||
|
@ -20,7 +22,7 @@ defmodule Farside.Server.UnHealthyCheck do
|
|||
def poll() do
|
||||
receive do
|
||||
after
|
||||
120_000 ->
|
||||
200_000 ->
|
||||
run()
|
||||
poll()
|
||||
end
|
||||
|
@ -29,6 +31,8 @@ defmodule Farside.Server.UnHealthyCheck do
|
|||
def run() do
|
||||
LastUpdated.value(DateTime.utc_now())
|
||||
|
||||
Logger.info("Unhealthy Service Check Running")
|
||||
|
||||
Registry.dispatch(:status, "unhealthy", fn entries ->
|
||||
for {pid, _} <- entries, do: GenServer.cast(pid, :check)
|
||||
end)
|
||||
|
|
Loading…
Add table
Add a link
Reference in a new issue