From 4f60a39d7f6aded5a7ccd5f003f77ed634415a90 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Mon, 8 Jan 2024 11:32:59 -0700 Subject: [PATCH 1/3] Replace libreddit w/ redlib, remove teddit Teddit is no longer maintained Libreddit has been forked to redlib, which seems to be actively trying to work around the changes to Reddit's API. Libreddit instances are now a mirror of redlib instances for the time being. --- .github/workflows/update-instances.yml | 32 +++++++---- lib/farside.ex | 4 +- services-full.json | 77 +++++++------------------- services.json | 70 ++++++++--------------- 4 files changed, 66 insertions(+), 117 deletions(-) diff --git a/.github/workflows/update-instances.yml b/.github/workflows/update-instances.yml index 3eb126f..f8e6080 100644 --- a/.github/workflows/update-instances.yml +++ b/.github/workflows/update-instances.yml @@ -161,18 +161,19 @@ jobs: # ============================================================== # teddit update + # Note: no longer maintained # ============================================================== - curl -s https://codeberg.org/teddit/teddit/raw/branch/main/instances.json | \ - jq '[ - .[] | - select(.url | contains("https")) | - .url ] | - sort' > teddit-tmp.json - jq --slurpfile teddit teddit-tmp.json \ - '( .[] | select(.type == "teddit") ) - .instances |= $teddit[0]' services-full.json > services-tmp.json + #curl -s https://codeberg.org/teddit/teddit/raw/branch/main/instances.json | \ + #jq '[ + #.[] | + #select(.url | contains("https")) | + #.url ] | + #sort' > teddit-tmp.json + #jq --slurpfile teddit teddit-tmp.json \ + #'( .[] | select(.type == "teddit") ) + #.instances |= $teddit[0]' services-full.json > services-tmp.json - apply_update + #apply_update # ============================================================== # Wikiless update @@ -191,18 +192,25 @@ jobs: #apply_update # ============================================================== - # libreddit update + # libreddit + redlib update # ============================================================== - curl -s https://raw.githubusercontent.com/libreddit/libreddit-instances/master/instances.json | \ + curl -s https://raw.githubusercontent.com/redlib-org/redlib-instances/main/instances.json | \ jq '[ .instances[] | select(.url) | .url ] | sort' > libreddit-tmp.json + jq --slurpfile libreddit libreddit-tmp.json \ '( .[] | select(.type == "libreddit") ) .instances |= $libreddit[0]' services-full.json > services-tmp.json + mv services-tmp.json services-full.json + + jq --slurpfile redlib libreddit-tmp.json \ + '( .[] | select(.type == "redlib") ) + .instances |= $redlib[0]' services-full.json > services-tmp.json + apply_update # ============================================================== diff --git a/lib/farside.ex b/lib/farside.ex index f31920a..fa74a80 100644 --- a/lib/farside.ex +++ b/lib/farside.ex @@ -7,7 +7,7 @@ defmodule Farside do # This enables Farside to redirect with links such as: # farside.link/https://www.youtube.com/watch?v=dQw4w9WgXcQ @youtube_regex ~r/youtu(.be|be.com)|invidious|piped/ - @reddit_regex ~r/reddit.com|libreddit|teddit/ + @reddit_regex ~r/reddit.com|libreddit|redlib/ @instagram_regex ~r/instagram.com|bibliogram/ @twitter_regex ~r/twitter.com|nitter/ @wikipedia_regex ~r/wikipedia.org|wikiless/ @@ -22,7 +22,7 @@ defmodule Farside do @parent_services %{ @youtube_regex => ["invidious", "piped"], - @reddit_regex => ["libreddit", "teddit"], + @reddit_regex => ["libreddit", "redlib"], @instagram_regex => ["bibliogram"], @twitter_regex => ["nitter"], @wikipedia_regex => ["wikiless"], diff --git a/services-full.json b/services-full.json index 86f8f72..45c4eab 100644 --- a/services-full.json +++ b/services-full.json @@ -2,68 +2,31 @@ { "type": "libreddit", "test_url": "/r/popular", - "fallback": "https://libreddit.spike.codes", + "fallback": "https://redlib.freedit.eu", "instances": [ - "https://discuss.whatever.social", "https://l.opnxng.com", - "https://libreddit.albr.to", - "https://libreddit.billbert.co", "https://libreddit.bus-hit.me", - "https://libreddit.cachyos.org", - "https://libreddit.dcs0.hu", - "https://libreddit.de", - "https://libreddit.domain.glass", - "https://libreddit.esmailelbob.xyz", - "https://libreddit.eu.org", - "https://libreddit.foss.wtf", - "https://libreddit.freedit.eu", - "https://libreddit.garudalinux.org", - "https://libreddit.gq", - "https://libreddit.hu", - "https://libreddit.kavin.rocks", - "https://libreddit.kutay.dev", - "https://libreddit.kylrth.com", - "https://libreddit.lunar.icu", - "https://libreddit.mha.fi", - "https://libreddit.nl", - "https://libreddit.northboot.xyz", - "https://libreddit.oxymagnesium.com", - "https://libreddit.privacy.com.de", - "https://libreddit.privacydev.net", "https://libreddit.projectsegfau.lt", - "https://libreddit.pufe.org", - "https://libreddit.pussthecat.org", - "https://libreddit.spike.codes", - "https://libreddit.strongthany.cc", - "https://libreddit.tiekoetter.com", - "https://libreddit.totaldarkness.net", - "https://libreddit.tux.pizza", - "https://lr.4201337.xyz", - "https://lr.aeong.one", - "https://lr.artemislena.eu", - "https://lr.mint.lgbt", - "https://lr.odyssey346.dev", - "https://lr.riverside.rocks", - "https://lr.slipfox.xyz", - "https://lr.vern.cc", - "https://r.ahwx.org", - "https://r.darklab.sh", - "https://r.nf", - "https://r.walkx.fyi", - "https://rd.777.tf", - "https://rd.funami.tech", - "https://reddi.tk", - "https://reddit.baby", - "https://reddit.dr460nf1r3.org", + "https://reddit.idevicehacked.com", "https://reddit.invak.id", - "https://reddit.leptons.xyz", - "https://reddit.moe.ngo", - "https://reddit.rtrace.io", - "https://reddit.simo.sh", - "https://reddit.smnz.de", - "https://reddit.thekitty.zone", - "https://safereddit.com", - "https://snoo.habedieeh.re" + "https://redlib.catsarch.com", + "https://redlib.freedit.eu", + "https://redlib.matthew.science" + ] + }, + { + "type": "redlib", + "test_url": "/r/popular", + "fallback": "https://redlib.freedit.eu", + "instances": [ + "https://l.opnxng.com", + "https://libreddit.bus-hit.me", + "https://libreddit.projectsegfau.lt", + "https://reddit.idevicehacked.com", + "https://reddit.invak.id", + "https://redlib.catsarch.com", + "https://redlib.freedit.eu", + "https://redlib.matthew.science" ] }, { diff --git a/services.json b/services.json index 946da47..38ad1c6 100644 --- a/services.json +++ b/services.json @@ -2,56 +2,29 @@ { "type": "libreddit", "test_url": "/r/popular", - "fallback": "https://libreddit.spike.codes", + "fallback": "https://redlib.freedit.eu", "instances": [ - "https://discuss.whatever.social", "https://l.opnxng.com", - "https://libreddit.albr.to", "https://libreddit.bus-hit.me", - "https://libreddit.dcs0.hu", - "https://libreddit.de", - "https://libreddit.esmailelbob.xyz", - "https://libreddit.foss.wtf", - "https://libreddit.gq", - "https://libreddit.hu", - "https://libreddit.kavin.rocks", - "https://libreddit.kutay.dev", - "https://libreddit.kylrth.com", - "https://libreddit.lunar.icu", - "https://libreddit.mha.fi", - "https://libreddit.nl", - "https://libreddit.northboot.xyz", - "https://libreddit.oxymagnesium.com", - "https://libreddit.privacy.com.de", - "https://libreddit.privacydev.net", "https://libreddit.projectsegfau.lt", - "https://libreddit.pufe.org", - "https://libreddit.pussthecat.org", - "https://libreddit.spike.codes", - "https://libreddit.strongthany.cc", - "https://libreddit.tiekoetter.com", - "https://libreddit.totaldarkness.net", - "https://libreddit.tux.pizza", - "https://lr.4201337.xyz", - "https://lr.artemislena.eu", - "https://lr.mint.lgbt", - "https://lr.odyssey346.dev", - "https://lr.riverside.rocks", - "https://lr.slipfox.xyz", - "https://lr.vern.cc", - "https://r.ahwx.org", - "https://r.darklab.sh", - "https://rd.777.tf", - "https://rd.funami.tech", - "https://reddit.baby", + "https://reddit.idevicehacked.com", "https://reddit.invak.id", - "https://reddit.leptons.xyz", - "https://reddit.rtrace.io", - "https://reddit.simo.sh", - "https://reddit.smnz.de", - "https://reddit.thekitty.zone", - "https://safereddit.com", - "https://snoo.habedieeh.re" + "https://redlib.catsarch.com", + "https://redlib.matthew.science" + ] + }, + { + "type": "redlib", + "test_url": "/r/popular", + "fallback": "https://redlib.freedit.eu", + "instances": [ + "https://l.opnxng.com", + "https://libreddit.bus-hit.me", + "https://libreddit.projectsegfau.lt", + "https://reddit.idevicehacked.com", + "https://reddit.invak.id", + "https://redlib.catsarch.com", + "https://redlib.matthew.science" ] }, { @@ -71,6 +44,7 @@ "https://teddit.bus-hit.me", "https://teddit.encrypted-data.xyz", "https://teddit.froth.zone", + "https://teddit.garudalinux.org", "https://teddit.ggc-project.de", "https://teddit.hostux.net", "https://teddit.manasiwibi.com", @@ -160,6 +134,7 @@ "https://nitter.1d4.us", "https://nitter.kavin.rocks", "https://nitter.unixfox.eu", + "https://n.actionsack.com", "https://nitter.moomoo.me", "https://bird.trom.tf", "https://nitter.it", @@ -215,7 +190,6 @@ "test_url": "/auto/en/hola", "fallback": "https://lingva.ml", "instances": [ - "https://lingva.ml", "https://translate.igna.rocks" ] }, @@ -238,12 +212,14 @@ "instances": [ "https://gowogle.voring.me", "https://s.tokhmi.xyz", + "https://search.dr460nf1r3.org", "https://search.sethforprivacy.com", "https://wg.vern.cc", "https://wgl.frail.duckdns.org", "https://whoogle.dcs0.hu", "https://whoogle.esmailelbob.xyz", "https://whoogle.hostux.net", + "https://whoogle.hxvy0.gq", "https://whoogle.lunar.icu", "https://whoogle.privacydev.net" ] @@ -376,6 +352,7 @@ "test_url": "/wiki/Wikipedia?lang=en", "fallback": "https://wikiless.org", "instances": [ + "https://wiki.604kph.xyz", "https://wiki.adminforge.de", "https://wiki.froth.zone", "https://wiki.slipfox.xyz", @@ -455,6 +432,7 @@ "https://quetre.odyssey346.dev", "https://quetre.privacydev.net", "https://ask.habedieeh.re", + "https://quetre.marcopisco.com", "https://quetre.blackdrgn.nl", "https://quetre.pufe.org", "https://quetre.lunar.icu", From 7f26ab3bbf3743e5f6924b51e08986b58641ba49 Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Mon, 8 Jan 2024 11:49:18 -0700 Subject: [PATCH 2/3] Return 429 for users exceeding 1 req/sec Farside has been getting used by some to rapidly scrape sites, which puts increased load and effort on maintainers of instances. Rather than funneling traffic towards the last selected instance, farside will now just return a 429 error when this behavior occurs. Closes #147 --- lib/farside/router.ex | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/lib/farside/router.ex b/lib/farside/router.ex index ce0eb0a..00d98e2 100644 --- a/lib/farside/router.ex +++ b/lib/farside/router.ex @@ -56,23 +56,21 @@ defmodule Farside.Router do Enum.join(glob, "/") end - instance = cond do + cond do conn.assigns[:throttle] != nil -> - Farside.get_service(service_name) - |> Farside.last_instance - |> Farside.amend_instance(service_name, path) + send_resp(conn, :too_many_requests, "Too many requests - max request rate is 1 per second") true -> - Farside.get_service(service_name) - |> Farside.pick_instance - |> Farside.amend_instance(service_name, path) - end + instance = Farside.get_service(service_name) + |> Farside.pick_instance + |> Farside.amend_instance(service_name, path) - # Redirect to the available instance - conn - |> Plug.Conn.resp(:found, "") - |> Plug.Conn.put_resp_header( - "location", - "#{instance}/#{path}#{get_query_params(conn)}" - ) + # Redirect to the available instance + conn + |> Plug.Conn.resp(:found, "") + |> Plug.Conn.put_resp_header( + "location", + "#{instance}/#{path}#{get_query_params(conn)}" + ) + end end end From 99b13b3aa6277f331e85e20cac7aed43a3c284fb Mon Sep 17 00:00:00 2001 From: Ben Busby Date: Mon, 8 Jan 2024 12:07:38 -0700 Subject: [PATCH 3/3] Replace bibliogram w/ proxigram Closes #149 --- lib/farside.ex | 22 ++-------------------- lib/farside/router.ex | 1 - services-full.json | 16 +++++++++------- services.json | 16 +++++++++------- 4 files changed, 20 insertions(+), 35 deletions(-) diff --git a/lib/farside.ex b/lib/farside.ex index fa74a80..d2398d5 100644 --- a/lib/farside.ex +++ b/lib/farside.ex @@ -8,7 +8,7 @@ defmodule Farside do # farside.link/https://www.youtube.com/watch?v=dQw4w9WgXcQ @youtube_regex ~r/youtu(.be|be.com)|invidious|piped/ @reddit_regex ~r/reddit.com|libreddit|redlib/ - @instagram_regex ~r/instagram.com|bibliogram/ + @instagram_regex ~r/instagram.com|proxigram/ @twitter_regex ~r/twitter.com|nitter/ @wikipedia_regex ~r/wikipedia.org|wikiless/ @medium_regex ~r/medium.com|scribe/ @@ -23,7 +23,7 @@ defmodule Farside do @parent_services %{ @youtube_regex => ["invidious", "piped"], @reddit_regex => ["libreddit", "redlib"], - @instagram_regex => ["bibliogram"], + @instagram_regex => ["proxigram"], @twitter_regex => ["nitter"], @wikipedia_regex => ["wikiless"], @medium_regex => ["scribe"], @@ -114,24 +114,6 @@ defmodule Farside do instance end - def amend_instance(instance, service, path) do - cond do - String.match?(service, @instagram_regex) -> - # Bibliogram doesn't have a 1:1 matching to Instagram URLs for users, - # so a "/u" is appended if the requested path doesn't explicitly include - # "/p" for a post or an empty path for the home page. - if String.length(path) > 0 and - !String.starts_with?(path, "p/") and - !String.starts_with?(path, "u/") do - "#{instance}/u" - else - instance - end - true -> - instance - end - end - def get_last_updated do CubDB.get(CubDB, "last_updated") end diff --git a/lib/farside/router.ex b/lib/farside/router.ex index 00d98e2..80fc82f 100644 --- a/lib/farside/router.ex +++ b/lib/farside/router.ex @@ -62,7 +62,6 @@ defmodule Farside.Router do true -> instance = Farside.get_service(service_name) |> Farside.pick_instance - |> Farside.amend_instance(service_name, path) # Redirect to the available instance conn diff --git a/services-full.json b/services-full.json index 45c4eab..4f7f593 100644 --- a/services-full.json +++ b/services-full.json @@ -65,15 +65,17 @@ ] }, { - "type": "bibliogram", + "type": "proxigram", "test_url": "/p/Ch2WRmiLMjj", - "fallback": "https://bibliogram.1d4.us", + "fallback": "https://ig.opnxng.com", "instances": [ - "https://bibliogram.1d4.us", - "https://bibliogram.froth.zone", - "https://bibliogram.priv.pw", - "https://ig.femboy.hu", - "https://ig.tokhmi.xyz" + "https://ig.opnxng.com", + "https://proxigram.lunar.icu", + "https://gram.whatever.social", + "https://ig.snine.nl", + "https://ig.floppa.one", + "https://proxigram.kyun.li", + "https://proxigram.ducks.party" ] }, { diff --git a/services.json b/services.json index 38ad1c6..76103c2 100644 --- a/services.json +++ b/services.json @@ -58,15 +58,17 @@ ] }, { - "type": "bibliogram", + "type": "proxigram", "test_url": "/p/Ch2WRmiLMjj", - "fallback": "https://bibliogram.1d4.us", + "fallback": "https://ig.opnxng.com", "instances": [ - "https://bibliogram.1d4.us", - "https://bibliogram.froth.zone", - "https://bibliogram.priv.pw", - "https://ig.femboy.hu", - "https://ig.tokhmi.xyz" + "https://ig.opnxng.com", + "https://proxigram.lunar.icu", + "https://gram.whatever.social", + "https://ig.snine.nl", + "https://ig.floppa.one", + "https://proxigram.kyun.li", + "https://proxigram.ducks.party" ] }, {