From b5bad4defc6c75b9b969658229ce5fd2f3a46107 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Tue, 21 Jan 2025 13:46:29 -0700 Subject: [PATCH] Rewrite project, add daily update of services list The project was rewritten from Elixir to Go, primarily because: - I don't write Elixir anymore and don't want to maintain a project in a language I no longer write - I already write Go for other projects, including my day job, so it's a safer bet for a project that I want to maintain long term - Go allows me to build portable executables that will make it easier for others to run farside on their own machines The Go version of Farsside also has a built in task to fetch the latest services{-full}.json file from the repo and ingest it, which makes running a farside server a lot simpler. It also automatically fetches the latest instance state from https://farside.link unless configured as a primary farside node, which will allow others to use farside without increasing traffic to all instances that are queried by farside (just to the farside node itself). --- .github/workflows/elixir.yml | 38 --------- .github/workflows/tests.yml | 19 +++++ .gitignore | 20 +---- README.md | 40 ++------- config/config.exs | 19 ----- config/runtime.exs | 6 -- db/cron.go | 139 ++++++++++++++++++++++++++++++ db/db.go | 150 +++++++++++++++++++++++++++++++++ db/db_test.go | 60 +++++++++++++ go.mod | 29 +++++++ go.sum | 141 +++++++++++++++++++++++++++++++ lib/farside.ex | 142 ------------------------------- lib/farside/application.ex | 29 ------- lib/farside/instances.ex | 134 ----------------------------- lib/farside/router.ex | 78 ----------------- lib/farside/scheduler.ex | 3 - lib/farside/server.ex | 25 ------ lib/farside/throttle.ex | 20 ----- lib/service.ex | 6 -- main.go | 37 ++++++++ mix.exs | 74 ---------------- mix.lock | 31 ------- route.eex | 10 --- index.eex => server/index.html | 24 +++--- server/route.html | 10 +++ server/server.go | 138 ++++++++++++++++++++++++++++++ server/server_test.go | 80 ++++++++++++++++++ services/mappings.go | 110 ++++++++++++++++++++++++ services/services.go | 93 ++++++++++++++++++++ test/farside_test.exs | 93 -------------------- test/test_helper.exs | 1 - 31 files changed, 1031 insertions(+), 768 deletions(-) delete mode 100644 .github/workflows/elixir.yml create mode 100644 .github/workflows/tests.yml delete mode 100644 config/config.exs delete mode 100644 config/runtime.exs create mode 100644 db/cron.go create mode 100644 db/db.go create mode 100644 db/db_test.go create mode 100644 go.mod create mode 100644 go.sum delete mode 100644 lib/farside.ex delete mode 100644 lib/farside/application.ex delete mode 100644 lib/farside/instances.ex delete mode 100644 lib/farside/router.ex delete mode 100644 lib/farside/scheduler.ex delete mode 100644 lib/farside/server.ex delete mode 100644 lib/farside/throttle.ex delete mode 100644 lib/service.ex create mode 100644 main.go delete mode 100644 mix.exs delete mode 100644 mix.lock delete mode 100644 route.eex rename index.eex => server/index.html (72%) create mode 100644 server/route.html create mode 100644 server/server.go create mode 100644 server/server_test.go create mode 100644 services/mappings.go create mode 100644 services/services.go delete mode 100644 test/farside_test.exs delete mode 100644 test/test_helper.exs diff --git a/.github/workflows/elixir.yml b/.github/workflows/elixir.yml deleted file mode 100644 index e42307f..0000000 --- a/.github/workflows/elixir.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Elixir CI - -on: - push: - branches: [ main ] - pull_request: - branches: [ main ] - -jobs: - build: - - name: Build and test - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - - name: Set up Elixir - uses: erlef/setup-beam@v1 - with: - elixir-version: '1.12.3' - otp-version: '24' - - - name: Restore dependencies cache - uses: actions/cache@v3 - with: - path: deps - key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} - restore-keys: ${{ runner.os }}-mix- - - - name: Install dependencies - run: mix deps.get - - - name: Initialize services - run: FARSIDE_TEST=1 FARSIDE_SERVICES_JSON=services-full.json mix run -e Farside.Instances.sync - - - name: Run tests - run: FARSIDE_TEST=1 FARSIDE_SERVICES_JSON=services-full.json mix test --trace diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..e0bd2fe --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,19 @@ +on: [push, pull_request] +name: Tests +jobs: + test: + strategy: + matrix: + go-version: [1.21.x, 1.22.x, 1.23.x] + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@v2 + - name: Test + run: go test -v ./... + diff --git a/.gitignore b/.gitignore index f83d69f..bf9bbe6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,18 +1,2 @@ -/_build -/cover -/deps -/doc -/.fetch -erl_crash.dump -*.ez -*.beam -/config/*.secret.exs -.elixir_ls/ - -# Ignore results from update script -.update-result* - -*.rdb -.idea/ -*.iml -*.cub +badger-db +farside diff --git a/README.md b/README.md index 7362269..303400a 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ [![Latest Release](https://img.shields.io/github/v/release/benbusby/farside?label=Release)](https://github.com/benbusby/farside/releases) [![MIT License](https://img.shields.io/github/license/benbusby/earthbound-themes.svg)](http://opensource.org/licenses/MIT) -[![Elixir CI](https://github.com/benbusby/privacy-revolver/actions/workflows/elixir.yml/badge.svg)](https://github.com/benbusby/privacy-revolver/actions/workflows/elixir.yml) +[![Tests](https://github.com/benbusby/farside/actions/workflows/tests.yml/badge.svg)](https://github.com/benbusby/farside/actions/workflows/tests.yml) @@ -24,7 +24,6 @@ Contents 3. [How It Works](#how-it-works) 4. [Cloudflare](#regarding-cloudflare) 5. [Development](#development) - 1. [Compiling](#compiling) 1. [Environment Variables](#environment-variables) ## About @@ -187,30 +186,9 @@ that their mission to centralize the entire web behind their service ultimately goes against what Farside is trying to solve. Use at your own discretion. ## Development -- Install [elixir](https://elixir-lang.org/install.html) -- (on Debian systems) Install [erlang-dev](https://packages.debian.org/sid/erlang-dev) -To run Farside without compiling, you can perform the following steps: - -- Install dependencies: `mix deps.get` -- Initialize db contents: `FARSIDE_CRON=0 mix run -e Farside.Instances.sync` -- Run Farside: `mix run --no-halt` - - Uses localhost:4001 - -### Compiling - -You can create a standalone Farside app using the steps below. In the example, the -Farside executable is copied to `/usr/local/bin`, but can be moved to any preferred -destination. Note that the executable still depends on the C runtime of the machine -it is built on, so if you want a more portable binary, you should build Farside on a -system with older library versions. - -``` -MIX_ENV=cli && mix deps.get && mix release -cp _build/cli/rel/bakeware/farside /usr/local/bin -sudo chmod +x /usr/local/bin/farside -farside -``` +- Install [Go](https://go.dev/doc/install) +- Compile with `go build` ### Environment Variables @@ -221,23 +199,23 @@ farside - + - - + + - - + + - +
FARSIDE_TESTIf enabled, bypasses the instance availability check and adds all instances to the pool.If enabled, bypasses the instance availability check and adds all instances to the pool
FARSIDE_PORT The port to run Farside on (default: `4001`)
FARSIDE_DATA_DIRThe path to the directory to use for storing instance data (default: `/tmp`)FARSIDE_DB_DIRThe path to the directory to use for storing instance data (default: `./`)
FARSIDE_SERVICES_JSONThe JSON file to use for selecting instances (default: `services.json`)FARSIDE_CF_ENABLEDSet to 1 to enable redirecting to instances behind cloudflare
FARSIDE_CRONSet to 0 to deactivate the scheduled instance availability check (default on).Set to 0 to deactivate the periodic instance availability check
diff --git a/config/config.exs b/config/config.exs deleted file mode 100644 index 11f61d9..0000000 --- a/config/config.exs +++ /dev/null @@ -1,19 +0,0 @@ -import Config - -config :farside, - update_file: ".update-results", - service_prefix: "service-", - fallback_suffix: "-fallback", - previous_suffix: "-previous", - index: "index.eex", - route: "route.eex", - headers: [ - {"User-Agent", "Mozilla/5.0 (compatible; Farside/0.1.0; +https://farside.link)"}, - {"Accept", "text/html"}, - {"Accept-Language", "en-US,en;q=0.5"}, - {"Accept-Encoding", "gzip, deflate, br"} - ], - queries: [ - "weather", - "time" - ] diff --git a/config/runtime.exs b/config/runtime.exs deleted file mode 100644 index e00feb8..0000000 --- a/config/runtime.exs +++ /dev/null @@ -1,6 +0,0 @@ -import Config - -config :farside, - port: System.get_env("FARSIDE_PORT", "4001"), - services_json: System.get_env("FARSIDE_SERVICES_JSON", "services.json"), - data_dir: System.get_env("FARSIDE_DATA_DIR", File.cwd!) diff --git a/db/cron.go b/db/cron.go new file mode 100644 index 0000000..a7282a4 --- /dev/null +++ b/db/cron.go @@ -0,0 +1,139 @@ +package db + +import ( + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" + "strings" + "time" + + "github.com/benbusby/farside/services" + "github.com/robfig/cron/v3" +) + +const defaultPrimary = "https://farside.link/state" +const defaultCFPrimary = "https://cf.farside.link/state" + +var LastUpdate time.Time + +func InitCronTasks() { + log.Println("Initializing cron tasks...") + + cronDisabled := os.Getenv("FARSIDE_CRON") + if len(cronDisabled) == 0 || cronDisabled == "1" { + c := cron.New() + c.AddFunc("@every 10m", queryServiceInstances) + c.AddFunc("@daily", updateServiceList) + c.Start() + } + + queryServiceInstances() +} + +func updateServiceList() { + fileName := services.GetServicesFileName() + _, _ = services.FetchServicesFile(fileName) + services.InitializeServices() +} + +func queryServiceInstances() { + log.Println("Starting instance queries...") + + isPrimary := os.Getenv("FARSIDE_PRIMARY") + if len(isPrimary) == 0 || isPrimary != "1" { + remoteServices, err := fetchInstancesFromPrimary() + if err != nil { + log.Println("Unable to fetch instances from primary", err) + } + + for _, service := range remoteServices { + SetInstances(service.Type, service.Instances) + } + + return + } + + for _, service := range services.ServiceList { + fmt.Printf("===== %s =====\n", service.Type) + var instances []string + for _, instance := range service.Instances { + testURL := strings.ReplaceAll( + service.TestURL, + "<%=query%>", + "current+weather") + available := queryServiceInstance( + instance, + testURL, + ) + + if available { + instances = append(instances, instance) + } + } + + SetInstances(service.Type, instances) + } + + LastUpdate = time.Now().UTC() +} + +func fetchInstancesFromPrimary() ([]services.Service, error) { + primaryURL := defaultPrimary + useCF := os.Getenv("FARSIDE_CF_ENABLED") + if len(useCF) > 0 && useCF == "1" { + primaryURL = defaultCFPrimary + } + + resp, err := http.Get(primaryURL) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + bodyBytes, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var serviceList []services.Service + err = json.Unmarshal(bodyBytes, &serviceList) + return serviceList, err +} + +func queryServiceInstance(instance, testURL string) bool { + testMode := os.Getenv("FARSIDE_TEST") + if len(testMode) > 0 && testMode == "1" { + return true + } + + ua := "Mozilla/5.0 (compatible; Farside/1.0.0; +https://farside.link)" + url := instance + testURL + + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + fmt.Println(" [ERRO] Failed to create new http request!", err) + return false + } + + req.Header.Set("User-Agent", ua) + client := &http.Client{ + Timeout: 10 * time.Second, + } + resp, err := client.Do(req) + + if err != nil { + fmt.Println(" [ERRO] Error fetching instance:", err) + return false + } else if resp.StatusCode != http.StatusOK { + fmt.Printf(" [WARN] Received non-200 status for %s\n", url) + return false + } else { + fmt.Printf(" [INFO] Received 200 status for %s\n", url) + } + + return true +} diff --git a/db/db.go b/db/db.go new file mode 100644 index 0000000..efdf145 --- /dev/null +++ b/db/db.go @@ -0,0 +1,150 @@ +package db + +import ( + "encoding/json" + "errors" + "log" + "math/rand" + "os" + "slices" + "time" + + "github.com/benbusby/farside/services" + "github.com/dgraph-io/badger/v4" +) + +var ( + badgerDB *badger.DB + selectionMap map[string]string + + cachedServiceList []services.Service + cacheUpdated time.Time +) + +func InitializeDB() error { + var err error + + dbDir := os.Getenv("FARSIDE_DB_DIR") + if len(dbDir) == 0 { + dbDir = "./badger-db" + } + + badgerDB, err = badger.Open(badger.DefaultOptions(dbDir)) + if err != nil { + return err + } + + return nil +} + +func SetInstances(service string, instances []string) error { + instancesBytes, err := json.Marshal(instances) + if err != nil { + return err + } + + err = badgerDB.Update(func(txn *badger.Txn) error { + err := txn.Set([]byte(service), instancesBytes) + return err + }) + + if err != nil { + return err + } + + return nil +} + +func GetInstance(service string) (string, error) { + instances, err := GetAllInstances(service) + if err != nil || len(instances) == 0 { + if err != nil { + log.Println("DB err:", err) + } + + link, ok := services.FallbackMap[service] + if !ok { + return "", errors.New("invalid service") + } + + return link, nil + } + + previous, ok := selectionMap[service] + if ok && len(instances) > 2 { + instances = slices.DeleteFunc(instances, func(i string) bool { + return i == previous + }) + } + + index := rand.Intn(len(instances)) + value := instances[index] + selectionMap[service] = value + return value, nil +} + +func GetAllInstances(service string) ([]string, error) { + var instances []string + err := badgerDB.View(func(txn *badger.Txn) error { + item, err := txn.Get([]byte(service)) + if err != nil { + return err + } + + err = item.Value(func(val []byte) error { + err := json.Unmarshal(val, &instances) + return err + }) + + return err + }) + + return instances, err +} + +func GetServiceList() []services.Service { + if cacheUpdated.Add(5 * time.Minute).After(time.Now().UTC()) { + return cachedServiceList + } + + canCache := true + + var serviceList []services.Service + for _, service := range services.ServiceList { + instances, err := GetAllInstances(service.Type) + if err != nil { + canCache = false + instances = []string{service.Fallback} + } + + storedService := services.Service{ + Type: service.Type, + Instances: instances, + } + + serviceList = append(serviceList, storedService) + } + + if canCache { + cachedServiceList = serviceList + cacheUpdated = time.Now().UTC() + } + + return serviceList +} + +func CloseDB() error { + log.Println("Closing database...") + err := badgerDB.Close() + if err != nil { + log.Println("Error closing database", err) + return err + } + + log.Println("Database closed!") + return nil +} + +func init() { + selectionMap = make(map[string]string) +} diff --git a/db/db_test.go b/db/db_test.go new file mode 100644 index 0000000..7bc92ca --- /dev/null +++ b/db/db_test.go @@ -0,0 +1,60 @@ +package db + +import ( + "log" + "os" + "slices" + "testing" +) + +func TestMain(m *testing.M) { + err := InitializeDB() + if err != nil { + log.Fatalln("Failed to initialize database") + } + + exitCode := m.Run() + + _ = CloseDB() + os.Exit(exitCode) +} + +func TestDatabase(t *testing.T) { + var ( + service = "test" + siteA = "a.com" + siteB = "b.com" + siteC = "c.com" + ) + instances := []string{siteA, siteB, siteC} + err := SetInstances(service, instances) + if err != nil { + t.Fatalf("Failed to set instances: %v\n", err) + } + + dbInstances, err := GetAllInstances(service) + if err != nil { + t.Fatalf("Failed to retrieve instances: %v\n", err) + } + + for _, instance := range instances { + idx := slices.Index(dbInstances, instance) + if idx < 0 { + t.Fatalf("Failed to find instance in list") + } + } + + firstInstance, err := GetInstance(service) + if err != nil { + t.Fatalf("Failed to fetch single instance: %v\n", err) + } + + secondInstance, err := GetInstance(service) + if err != nil { + t.Fatalf("Failed to fetch single instance (second): %v\n", err) + } else if firstInstance == secondInstance { + t.Fatalf("Same instance was selected twice") + } + + _ = CloseDB() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..8375e7c --- /dev/null +++ b/go.mod @@ -0,0 +1,29 @@ +module github.com/benbusby/farside + +go 1.23.4 + +require ( + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charmbracelet/lipgloss v0.10.0 // indirect + github.com/dgraph-io/badger/v4 v4.5.0 // indirect + github.com/dgraph-io/ristretto/v2 v2.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/klauspost/compress v1.17.11 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/mattn/go-isatty v0.0.18 // indirect + github.com/mattn/go-runewidth v0.0.15 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.15.2 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/robfig/cron/v3 v3.0.0 // indirect + go.opencensus.io v0.24.0 // indirect + golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect + golang.org/x/net v0.31.0 // indirect + golang.org/x/sys v0.27.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..55e9b64 --- /dev/null +++ b/go.sum @@ -0,0 +1,141 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/lipgloss v0.10.0 h1:KWeXFSexGcfahHX+54URiZGkBFazf70JNMtwg/AFW3s= +github.com/charmbracelet/lipgloss v0.10.0/go.mod h1:Wig9DSfvANsxqkRsqj6x87irdy123SR4dOXlKa91ciE= +github.com/charmbracelet/log v0.4.0 h1:G9bQAcx8rWA2T3pWvx7YtPTPwgqpk7D68BX21IRW8ZM= +github.com/charmbracelet/log v0.4.0/go.mod h1:63bXt/djrizTec0l11H20t8FDSvA4CRZJ1KH22MdptM= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/badger/v4 v4.5.0 h1:TeJE3I1pIWLBjYhIYCA1+uxrjWEoJXImFBMEBVSm16g= +github.com/dgraph-io/badger/v4 v4.5.0/go.mod h1:ysgYmIeG8dS/E8kwxT7xHyc7MkmwNYLRoYnFbr7387A= +github.com/dgraph-io/ristretto/v2 v2.0.0 h1:l0yiSOtlJvc0otkqyMaDNysg8E9/F/TYZwMbxscNOAQ= +github.com/dgraph-io/ristretto/v2 v2.0.0/go.mod h1:FVFokF2dRqXyPyeMnK1YDy8Fc6aTe0IKgbcd03CYeEk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= +github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mattn/go-isatty v0.0.18 h1:DOKFKCQ7FNG2L1rbrmstDN4QVRdS89Nkh85u68Uwp98= +github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U= +github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.15.2 h1:GohcuySI0QmI3wN8Ok9PtKGkgkFIk7y6Vpb5PvrY+Wo= +github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1noLXviQ8= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/robfig/cron/v3 v3.0.0 h1:kQ6Cb7aHOHTSzNVNEhmp8EcWKLb4CbiMW9h9VyIhO4E= +github.com/robfig/cron/v3 v3.0.0/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo= +golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s= +golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/lib/farside.ex b/lib/farside.ex deleted file mode 100644 index 348f77c..0000000 --- a/lib/farside.ex +++ /dev/null @@ -1,142 +0,0 @@ - defmodule Farside do - @service_prefix Application.compile_env!(:farside, :service_prefix) - @fallback_suffix Application.compile_env!(:farside, :fallback_suffix) - @previous_suffix Application.compile_env!(:farside, :previous_suffix) - - # Define relation between available services and their parent service. - # This enables Farside to redirect with links such as: - # farside.link/https://www.youtube.com/watch?v=dQw4w9WgXcQ - @youtube_regex ~r/youtu(.be|be.com)|invidious|piped/ - @twitter_regex ~r/twitter.com|x.com|nitter/ - @reddit_regex ~r/reddit.com|libreddit|redlib/ - @instagram_regex ~r/instagram.com|proxigram/ - @wikipedia_regex ~r/wikipedia.org|wikiless/ - @medium_regex ~r/medium.com|scribe/ - @odysee_regex ~r/odysee.com|librarian/ - @imgur_regex ~r/imgur.com|rimgo/ - @gtranslate_regex ~r/translate.google.com|lingva/ - @tiktok_regex ~r/tiktok.com|proxitok/ - @imdb_regex ~r/imdb.com|libremdb/ - @quora_regex ~r/quora.com|quetre/ - @gsearch_regex ~r/google.com\/search|whoogle/ - @fandom_regex ~r/fandom.com|breezewiki/ - @github_regex ~r/github.com|gothub/ - @stackoverflow_regex ~r/stackoverflow.com|anonymousoverflow/ - - @parent_services %{ - @youtube_regex => ["invidious", "piped"], - @reddit_regex => ["libreddit", "redlib"], - @instagram_regex => ["proxigram"], - @twitter_regex => ["nitter"], - @wikipedia_regex => ["wikiless"], - @medium_regex => ["scribe"], - @odysee_regex => ["librarian"], - @imgur_regex => ["rimgo"], - @gtranslate_regex => ["lingva"], - @tiktok_regex => ["proxitok"], - @imdb_regex => ["libremdb"], - @quora_regex => ["quetre"], - @gsearch_regex => ["whoogle"], - @fandom_regex => ["breezewiki"], - @github_regex => ["gothub"], - @stackoverflow_regex => ["anonymousoverflow"] - } - - def get_services_map do - service_list = CubDB.select(CubDB) - |> Stream.map(fn {key, _value} -> key end) - |> Stream.filter(fn key -> String.starts_with?(key, @service_prefix) end) - |> Enum.to_list - - # Match service name to list of available instances - Enum.reduce(service_list, %{}, fn service, acc -> - instance_list = CubDB.get(CubDB, service) - - Map.put( - acc, - String.replace_prefix( - service, - @service_prefix, - "" - ), - instance_list - ) - end) - end - - def get_service(service) do - # Check if service has an entry in the db, otherwise try to - # match against available parent services - service_name = cond do - !check_service(service) -> - Enum.find_value( - @parent_services, - fn {k, v} -> - String.match?(service, k) && Enum.random(v) - end) - true -> - service - end - - service_name - end - - def check_service(service) do - # Checks to see if a specific service has instances available - instances = CubDB.get(CubDB, "#{@service_prefix}#{service}") - - instances != nil && Enum.count(instances) > 0 - end - - def last_instance(service) do - # Fetches the last selected instance for a particular service - CubDB.get(CubDB, "#{service}#{@previous_suffix}") - end - - def pick_instance(service) do - instances = CubDB.get(CubDB, "#{@service_prefix}#{service}") - - # Either pick a random available instance, - # or fall back to the default one - instance = - if instances != nil && Enum.count(instances) > 0 do - if Enum.count(instances) == 1 do - # If there's only one instance, just return that one... - List.first(instances) - else - # ...otherwise pick a random one from the list, ensuring - # that the same instance is never picked twice in a row. - instance = - Enum.filter(instances, &(&1 != last_instance(service))) - |> Enum.random() - - CubDB.put(CubDB, "#{service}#{@previous_suffix}", instance) - - instance - end - else - CubDB.get(CubDB, "#{service}#{@fallback_suffix}") - end - instance - end - - def amend_instance(instance, service, path) do - cond do - String.match?(service, @fandom_regex) -> - # Fandom links require the subdomain to be preserved, otherwise the - # requested path won't work. - if String.contains?(service, ".fandom.com") do - wiki = String.replace(service, ".fandom.com", "") - "#{instance}/#{wiki}" - else - instance - end - true -> - instance - end - end - - def get_last_updated do - CubDB.get(CubDB, "last_updated") - end -end diff --git a/lib/farside/application.ex b/lib/farside/application.ex deleted file mode 100644 index 0f11f73..0000000 --- a/lib/farside/application.ex +++ /dev/null @@ -1,29 +0,0 @@ -defmodule Farside.Application do - @moduledoc false - - use Application - - @impl true - def start(_type, _args) do - farside_port = Application.fetch_env!(:farside, :port) - data_dir = Application.fetch_env!(:farside, :data_dir) - IO.puts "Running on http://localhost:#{farside_port}" - - children = [ - Plug.Cowboy.child_spec( - scheme: :http, - plug: Farside.Router, - options: [ - port: String.to_integer(farside_port) - ] - ), - {PlugAttack.Storage.Ets, name: Farside.Throttle.Storage, clean_period: 60_000}, - {CubDB, [data_dir: data_dir, name: CubDB, auto_compact: true]}, - Farside.Scheduler, - Farside.Server - ] - - opts = [strategy: :one_for_one, name: Farside.Supervisor] - Supervisor.start_link(children, opts) - end -end diff --git a/lib/farside/instances.ex b/lib/farside/instances.ex deleted file mode 100644 index d0d26f6..0000000 --- a/lib/farside/instances.ex +++ /dev/null @@ -1,134 +0,0 @@ -defmodule Farside.Instances do - @fallback_suffix Application.fetch_env!(:farside, :fallback_suffix) - @update_file Application.fetch_env!(:farside, :update_file) - @service_prefix Application.fetch_env!(:farside, :service_prefix) - @headers Application.fetch_env!(:farside, :headers) - @queries Application.fetch_env!(:farside, :queries) - @debug_header "======== " - @debug_spacer " " - - # These instance uptimes are inspected as part of the nightly Farside build, - # and should not be included in the constant periodic update. - @skip_service_updates ["searxng", "nitter"] - - def sync() do - File.rename(@update_file, "#{@update_file}-prev") - update() - - # Add UTC time of last update - CubDB.put(CubDB, "last_updated", Calendar.strftime(DateTime.utc_now(), "%c")) - end - - def request(url) do - IO.puts("#{@debug_spacer}#{url}") - - cond do - System.get_env("FARSIDE_TEST") -> - :good - - true -> - HTTPoison.get(url, @headers) - |> then(&elem(&1, 1)) - |> Map.get(:status_code) - |> case do - n when n < 300 -> - IO.puts("#{@debug_spacer}✓ [#{n}]") - :good - - n -> - IO.puts("#{@debug_spacer}x [#{(n && n) || "error"}]") - :bad - end - end - end - - def update() do - services_json = Application.fetch_env!(:farside, :services_json) - {:ok, file} = File.read(services_json) - {:ok, json} = Jason.decode(file) - - # Loop through all instances and check each for availability - for service_json <- json do - service_atom = for {key, val} <- service_json, into: %{} do - {String.to_existing_atom(key), val} - end - - service = struct(%Service{}, service_atom) - - IO.puts("#{@debug_header}#{service.type}") - - result = cond do - Enum.member?(@skip_service_updates, service.type) -> - get_service_vals(service.instances) - true -> - Enum.filter(service.instances, fn instance_url -> - test_url = get_test_val(instance_url) - test_path = get_test_val(service.test_url) - test_request_url = gen_validation_url(test_url, test_path) - - service_url = get_service_val(instance_url) - service_path = get_service_val(service.test_url) - service_request_url = gen_validation_url(service_url, service_path) - - cond do - service_url != test_url -> - service_up = request(service_request_url) - test_up = request(test_request_url) - - service_up == :good && test_up == :good - true -> - request(test_request_url) == :good - end - end) - end - - add_to_db(service, result) - log_results(service.type, result) - end - end - - def add_to_db(service, instances) do - # Ensure only service URLs are inserted, not test URLs (separated by "|") - instances = get_service_vals(instances) - - # Remove previous list of instances - CubDB.delete(CubDB, "#{@service_prefix}#{service.type}") - - # Update with new list of available instances - CubDB.put(CubDB, "#{@service_prefix}#{service.type}", instances) - - # Set fallback to one of the available instances, - # or the default instance if all are "down" - if Enum.count(instances) > 0 do - CubDB.put(CubDB, "#{service.type}#{@fallback_suffix}", Enum.random(instances)) - else - CubDB.put(CubDB, "#{service.type}#{@fallback_suffix}", service.fallback) - end - end - - def log_results(service_name, results) do - {:ok, file} = File.open(@update_file, [:append, {:delayed_write, 100, 20}]) - IO.write(file, "#{service_name}: #{inspect(results)}\n") - File.close(file) - end - - def gen_validation_url(url, path) do - url <> EEx.eval_string(path, query: Enum.random(@queries)) - end - - def get_service_vals(services) do - Enum.map(services, fn x -> get_service_val(x) end) - end - - def get_service_val(service) do - String.split(service, "|") |> List.first - end - - def get_test_vals(services) do - Enum.map(services, fn x -> get_test_val(x) end) - end - - def get_test_val(service) do - String.split(service, "|") |> List.last - end -end diff --git a/lib/farside/router.ex b/lib/farside/router.ex deleted file mode 100644 index ec49862..0000000 --- a/lib/farside/router.ex +++ /dev/null @@ -1,78 +0,0 @@ -defmodule Farside.Router do - @index Application.fetch_env!(:farside, :index) - @route Application.fetch_env!(:farside, :route) - - use Plug.Router - - plug(RemoteIp) - plug(Farside.Throttle) - plug(:match) - plug(:dispatch) - - def get_query_params(conn) do - cond do - String.length(conn.query_string) > 0 -> - "?#{conn.query_string}" - - true -> - "" - end - end - - match "/" do - resp = - EEx.eval_file( - @index, - last_updated: Farside.get_last_updated(), - services: Farside.get_services_map() - ) - - put_resp_header(conn, "content-type", "text/html") - |> send_resp(200, resp) - end - - match "/_/:service/*glob" do - r_path = String.slice(conn.request_path, 2..-1) - - resp = - EEx.eval_file( - @route, - instance_url: "#{r_path}#{get_query_params(conn)}" - ) - - send_resp(conn, 200, resp) - end - - match "/:service/*glob" do - service_name = cond do - service =~ "http" -> - List.first(glob) - true -> - service - end - - path = cond do - service_name != service -> - Enum.join(Enum.slice(glob, 1..-1), "/") - true -> - Enum.join(glob, "/") - end - - cond do - conn.assigns[:throttle] != nil -> - send_resp(conn, :too_many_requests, "Too many requests - max request rate is 1 per second") - true -> - instance = Farside.get_service(service_name) - |> Farside.pick_instance - |> Farside.amend_instance(service_name, path) - - # Redirect to the available instance - conn - |> Plug.Conn.resp(:found, "") - |> Plug.Conn.put_resp_header( - "location", - "#{instance}/#{path}#{get_query_params(conn)}" - ) - end - end -end diff --git a/lib/farside/scheduler.ex b/lib/farside/scheduler.ex deleted file mode 100644 index 4707624..0000000 --- a/lib/farside/scheduler.ex +++ /dev/null @@ -1,3 +0,0 @@ -defmodule Farside.Scheduler do - use Quantum, otp_app: :farside -end diff --git a/lib/farside/server.ex b/lib/farside/server.ex deleted file mode 100644 index cdb6682..0000000 --- a/lib/farside/server.ex +++ /dev/null @@ -1,25 +0,0 @@ -defmodule Farside.Server do - use GenServer - import Crontab.CronExpression - - def init(init_arg) do - {:ok, init_arg} - end - - def start_link(arg) do - test = System.get_env("FARSIDE_TEST") - cron = System.get_env("FARSIDE_CRON") - - if test == "1" || cron == "0" do - IO.puts("Skipping sync job setup...") - else - Farside.Scheduler.new_job() - |> Quantum.Job.set_name(:sync) - |> Quantum.Job.set_schedule(~e[*/5 * * * *]) - |> Quantum.Job.set_task(fn -> Farside.Instances.sync() end) - |> Farside.Scheduler.add_job() - end - - GenServer.start_link(__MODULE__, arg) - end -end diff --git a/lib/farside/throttle.ex b/lib/farside/throttle.ex deleted file mode 100644 index e2561ab..0000000 --- a/lib/farside/throttle.ex +++ /dev/null @@ -1,20 +0,0 @@ -defmodule Farside.Throttle do - import Plug.Conn - use PlugAttack - - rule "throttle per ip", conn do - # throttle to 1 request per second - throttle(conn.remote_ip, - period: 1_000, - limit: 1, - storage: {PlugAttack.Storage.Ets, Farside.Throttle.Storage} - ) - end - - def allow_action(conn, _data, _opts), do: conn - - def block_action(conn, _data, _opts) do - conn = assign(conn, :throttle, 1) - conn - end -end diff --git a/lib/service.ex b/lib/service.ex deleted file mode 100644 index ae964f6..0000000 --- a/lib/service.ex +++ /dev/null @@ -1,6 +0,0 @@ -defmodule Service do - defstruct type: nil, - test_url: nil, - fallback: nil, - instances: [] -end diff --git a/main.go b/main.go new file mode 100644 index 0000000..2ca28ec --- /dev/null +++ b/main.go @@ -0,0 +1,37 @@ +package main + +import ( + "log" + "os" + "os/signal" + "syscall" + + "github.com/benbusby/farside/db" + "github.com/benbusby/farside/server" + "github.com/benbusby/farside/services" +) + +func main() { + err := db.InitializeDB() + if err != nil { + log.Fatal(err) + } + + err = services.InitializeServices() + if err != nil { + log.Fatal(err) + } + + db.InitCronTasks() + + signalChan := make(chan os.Signal, 1) + signal.Notify(signalChan, os.Interrupt, syscall.SIGTERM) + + go func() { + <-signalChan + _ = db.CloseDB() + os.Exit(0) + }() + + server.RunServer() +} diff --git a/mix.exs b/mix.exs deleted file mode 100644 index c2ffd56..0000000 --- a/mix.exs +++ /dev/null @@ -1,74 +0,0 @@ -defmodule Farside.MixProject do - use Mix.Project - - @source_url "https://github.com/benbusby/farside.git" - @version "0.1.1" - @app :farside - - def project do - [ - app: @app, - version: @version, - name: "farside", - elixir: "~> 1.8", - source_url: @source_url, - start_permanent: Mix.env() == :prod || Mix.env() == :cli, - deps: deps(), - aliases: aliases(), - description: description(), - package: package(), - releases: [{@app, release()}], - preferred_cli_env: [release: :cli] - ] - end - - defp aliases do - [] - end - - # Run "mix help compile.app" to learn about applications. - def application do - [ - extra_applications: [:logger], - mod: {Farside.Application, []} - ] - end - - # Run "mix help deps" to learn about dependencies. - defp deps do - [ - {:httpoison, "~> 1.8"}, - {:jason, "~> 1.1"}, - {:plug_attack, "~> 0.4.2"}, - {:plug_cowboy, "~> 2.0"}, - {:quantum, "~> 3.0"}, - {:remote_ip, "~> 1.1"}, - {:cubdb, "~> 2.0.1"}, - {:bakeware, "~> 0.2.4"} - ] - end - - defp description() do - "A redirecting service for FOSS alternative frontends." - end - - defp package() do - [ - name: "farside", - files: ["lib", "mix.exs", "README*"], - maintainers: ["Ben Busby"], - licenses: ["MIT"], - links: %{"GitHub" => "https://github.com/benbusby/farside"} - ] - end - - defp release() do - [ - overwrite: true, - cookie: "#{@app}_cookie", - quiet: true, - steps: [:assemble, &Bakeware.assemble/1], - strip_beams: Mix.env() == :cli - ] - end -end diff --git a/mix.lock b/mix.lock deleted file mode 100644 index 810d34d..0000000 --- a/mix.lock +++ /dev/null @@ -1,31 +0,0 @@ -%{ - "bakeware": {:hex, :bakeware, "0.2.4", "0aaf49b34f4bab2aa433f9ff1485d9401e421603160abd6d269c469fc7b65212", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "7b97bcf6fbeee53bb32441d6c495bf478d26f9575633cfef6831e421e86ada6d"}, - "certifi": {:hex, :certifi, "2.9.0", "6f2a475689dd47f19fb74334859d460a2dc4e3252a3324bd2111b8f0429e7e21", [:rebar3], [], "hexpm", "266da46bdb06d6c6d35fde799bcb28d36d985d424ad7c08b5bb48f5b5cdd4641"}, - "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, - "cowboy": {:hex, :cowboy, "2.9.0", "865dd8b6607e14cf03282e10e934023a1bd8be6f6bacf921a7e2a96d800cd452", [:make, :rebar3], [{:cowlib, "2.11.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "2c729f934b4e1aa149aff882f57c6372c15399a20d54f65c8d67bef583021bde"}, - "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.11.0", "0b9ff9c346629256c42ebe1eeb769a83c6cb771a6ee5960bd110ab0b9b872063", [:make, :rebar3], [], "hexpm", "2b3e9da0b21c4565751a6d4901c20d1b4cc25cbb7fd50d91d2ab6dd287bc86a9"}, - "crontab": {:hex, :crontab, "1.1.11", "4028ced51b813a5061f85b689d4391ef0c27550c8ab09aaf139e4295c3d93ea4", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "ecb045f9ac14a3e2990e54368f70cdb6e2f2abafc5bc329d6c31f0c74b653787"}, - "cubdb": {:hex, :cubdb, "2.0.1", "24cab8fb4128df704c52ed641f5ed70af352f7a3a80cebbb44c3bbadc3fd5f45", [:mix], [], "hexpm", "57cf25aebfc34f4580d9075da06882b4fe3e0739f5353d4dcc213e9cc1b10cdf"}, - "elixir_make": {:hex, :elixir_make, "0.7.6", "67716309dc5d43e16b5abbd00c01b8df6a0c2ab54a8f595468035a50189f9169", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}], "hexpm", "5a0569756b0f7873a77687800c164cca6dfc03a09418e6fcf853d78991f49940"}, - "gen_stage": {:hex, :gen_stage, "1.1.2", "b1656cd4ba431ed02c5656fe10cb5423820847113a07218da68eae5d6a260c23", [:mix], [], "hexpm", "9e39af23140f704e2b07a3e29d8f05fd21c2aaf4088ff43cb82be4b9e3148d02"}, - "hackney": {:hex, :hackney, "1.18.1", "f48bf88f521f2a229fc7bae88cf4f85adc9cd9bcf23b5dc8eb6a1788c662c4f6", [:rebar3], [{:certifi, "~>2.9.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~>6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~>1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~>1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.3.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~>1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a4ecdaff44297e9b5894ae499e9a070ea1888c84afdd1fd9b7b2bc384950128e"}, - "httpoison": {:hex, :httpoison, "1.8.2", "9eb9c63ae289296a544842ef816a85d881d4a31f518a0fec089aaa744beae290", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "2bb350d26972e30c96e2ca74a1aaf8293d61d0742ff17f01e0279fef11599921"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~>0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, - "jason": {:hex, :jason, "1.4.0", "e855647bc964a44e2f67df589ccf49105ae039d4179db7f6271dfd3843dc27e6", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "79a3791085b2a0f743ca04cec0f7be26443738779d09302e01318f97bdb82121"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "2.0.3", "3676436d3d1f7b81b5a2d2bd8405f412c677558c81b1c92be58c00562bb59095", [:mix], [], "hexpm", "27a30bf0db44d25eecba73755acf4068cbfe26a4372f9eb3e4ea3a45956bff6b"}, - "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "parse_trans": {:hex, :parse_trans, "3.3.1", "16328ab840cc09919bd10dab29e431da3af9e9e7e7e6f0089dd5a2d2820011d8", [:rebar3], [], "hexpm", "07cd9577885f56362d414e8c4c4e6bdf10d43a8767abb92d24cbe8b24c54888b"}, - "plug": {:hex, :plug, "1.14.0", "ba4f558468f69cbd9f6b356d25443d0b796fbdc887e03fa89001384a9cac638f", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "bf020432c7d4feb7b3af16a0c2701455cbbbb95e5b6866132cb09eb0c29adc14"}, - "plug_attack": {:hex, :plug_attack, "0.4.3", "88e6c464d68b1491aa083a0347d59d58ba71a7e591a7f8e1b675e8c7792a0ba8", [:mix], [{:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "9ed6fb8a6f613a36040f2875130a21187126c5625092f24bc851f7f12a8cbdc1"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.6.0", "d1cf12ff96a1ca4f52207c5271a6c351a4733f413803488d75b70ccf44aebec2", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "073cf20b753ce6682ed72905cd62a2d4bd9bad1bf9f7feb02a1b8e525bd94fa6"}, - "plug_crypto": {:hex, :plug_crypto, "1.2.3", "8f77d13aeb32bfd9e654cb68f0af517b371fb34c56c9f2b58fe3df1235c1251a", [:mix], [], "hexpm", "b5672099c6ad5c202c45f5a403f21a3411247f164e4a8fab056e5cd8a290f4a2"}, - "quantum": {:hex, :quantum, "3.5.0", "8d2c5ba68c55991e8975aca368e3ab844ba01f4b87c4185a7403280e2c99cf34", [:mix], [{:crontab, "~> 1.1", [hex: :crontab, repo: "hexpm", optional: false]}, {:gen_stage, "~> 0.14 or ~> 1.0", [hex: :gen_stage, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_registry, "~> 0.2", [hex: :telemetry_registry, repo: "hexpm", optional: false]}], "hexpm", "cab737d1d9779f43cb1d701f46dd05ea58146fd96238d91c9e0da662c1982bb6"}, - "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, - "remote_ip": {:hex, :remote_ip, "1.1.0", "cb308841595d15df3f9073b7c39243a1dd6ca56e5020295cb012c76fbec50f2d", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "616ffdf66aaad6a72fc546dabf42eed87e2a99e97b09cbd92b10cc180d02ed74"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, - "telemetry": {:hex, :telemetry, "1.1.0", "a589817034a27eab11144ad24d5c0f9fab1f58173274b1e9bae7074af9cbee51", [:rebar3], [], "hexpm", "b727b2a1f75614774cff2d7565b64d0dfa5bd52ba517f16543e6fc7efcc0df48"}, - "telemetry_registry": {:hex, :telemetry_registry, "0.3.0", "6768f151ea53fc0fbca70dbff5b20a8d663ee4e0c0b2ae589590e08658e76f1e", [:mix, :rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "492e2adbc609f3e79ece7f29fec363a97a2c484ac78a83098535d6564781e917"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, -} diff --git a/route.eex b/route.eex deleted file mode 100644 index 55538c7..0000000 --- a/route.eex +++ /dev/null @@ -1,10 +0,0 @@ - - Farside Redirect - - - - - Redirecting to <%= instance_url %>... - diff --git a/index.eex b/server/index.html similarity index 72% rename from index.eex rename to server/index.html index c9f4a50..0455489 100644 --- a/index.eex +++ b/server/index.html @@ -1,3 +1,6 @@ + + + Farside