mirrors/farside
1
0
Fork 0
mirror of https://github.com/benbusby/farside.git synced 2025-04-20 10:58:42 +00:00
Code Wiki Activity

Update reddit + ig frontends, update rate limiting

Browse source 
Farside ratelimiting has been updated to return a 429 when a user
exceeds 1 request/sec. This should help eliminate a lot of scraping type
behavior that instance maintainers have been dealing with from Farside
lately.

Service changes:
- Teddit removed (not maintained)
- Bibliogram replaced by Proxigram
- Libreddit merged with redlib
This commit is contained in:
Ben Busby 2024-01-08 12:37:17 -07:00
commit 247114d328
No known key found for this signature in database
GPG key ID: B9B7231E01D924A1
5 changed files with 101 additions and 74 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

33
.github/workflows/update-instances.yml vendored
View file

@ -166,19 +166,19 @@ jobs:
# ============================================================== # ==============================================================
# teddit update # teddit update
# Note: no longer maintained
# ============================================================== # ==============================================================
curl -s https://codeberg.org/teddit/teddit/raw/branch/main/instances.json | \ #curl -s https://codeberg.org/teddit/teddit/raw/branch/main/instances.json | \
jq '[ #jq '[
.[] | #.[] |
select(.url != null) | #select(.url | contains("https")) |
select(.url | contains("https")) | #.url ] |
.url ] | #sort' > teddit-tmp.json
sort' > teddit-tmp.json #jq --slurpfile teddit teddit-tmp.json \
jq --slurpfile teddit teddit-tmp.json \ #'( .[] | select(.type == "teddit") )
'( .[] | select(.type == "teddit") ) #.instances |= $teddit[0]' services-full.json > services-tmp.json
.instances |= $teddit[0]' services-full.json > services-tmp.json
apply_update #apply_update
# ============================================================== # ==============================================================
# Wikiless update # Wikiless update
@ -197,18 +197,25 @@ jobs:
#apply_update #apply_update
# ============================================================== # ==============================================================
# libreddit update # libreddit + redlib update
# ============================================================== # ==============================================================
curl -s https://raw.githubusercontent.com/libreddit/libreddit-instances/master/instances.json | \ curl -s https://raw.githubusercontent.com/redlib-org/redlib-instances/main/instances.json | \
jq '[ jq '[
.instances[] | .instances[] |
select(.url) | select(.url) |
.url ] | .url ] |
sort' > libreddit-tmp.json sort' > libreddit-tmp.json
jq --slurpfile libreddit libreddit-tmp.json \ jq --slurpfile libreddit libreddit-tmp.json \
'( .[] | select(.type == "libreddit") ) '( .[] | select(.type == "libreddit") )
.instances |= $libreddit[0]' services-full.json > services-tmp.json .instances |= $libreddit[0]' services-full.json > services-tmp.json
mv services-tmp.json services-full.json
jq --slurpfile redlib libreddit-tmp.json \
'( .[] | select(.type == "redlib") )
.instances |= $redlib[0]' services-full.json > services-tmp.json
apply_update apply_update
# ============================================================== # ==============================================================

19
lib/farside.ex
View file

@ -7,9 +7,9 @@ defmodule Farside do
# This enables Farside to redirect with links such as: # This enables Farside to redirect with links such as:
# farside.link/https://www.youtube.com/watch?v=dQw4w9WgXcQ # farside.link/https://www.youtube.com/watch?v=dQw4w9WgXcQ
@youtube_regex ~r/youtu(.be|be.com)|invidious|piped/ @youtube_regex ~r/youtu(.be|be.com)|invidious|piped/
@reddit_regex ~r/reddit.com|libreddit|teddit/
@instagram_regex ~r/instagram.com|bibliogram/
@twitter_regex ~r/twitter.com|x.com|nitter/ @twitter_regex ~r/twitter.com|x.com|nitter/
@reddit_regex ~r/reddit.com|libreddit|redlib/
@instagram_regex ~r/instagram.com|proxigram/
@wikipedia_regex ~r/wikipedia.org|wikiless/ @wikipedia_regex ~r/wikipedia.org|wikiless/
@medium_regex ~r/medium.com|scribe/ @medium_regex ~r/medium.com|scribe/
@odysee_regex ~r/odysee.com|librarian/ @odysee_regex ~r/odysee.com|librarian/
@ -25,8 +25,8 @@ defmodule Farside do
@parent_services %{ @parent_services %{
@youtube_regex => ["invidious", "piped"], @youtube_regex => ["invidious", "piped"],
@reddit_regex => ["libreddit", "teddit"], @reddit_regex => ["libreddit", "redlib"],
@instagram_regex => ["bibliogram"], @instagram_regex => ["proxigram"],
@twitter_regex => ["nitter"], @twitter_regex => ["nitter"],
@wikipedia_regex => ["wikiless"], @wikipedia_regex => ["wikiless"],
@medium_regex => ["scribe"], @medium_regex => ["scribe"],
@ -122,17 +122,6 @@ defmodule Farside do
def amend_instance(instance, service, path) do def amend_instance(instance, service, path) do
cond do cond do
String.match?(service, @instagram_regex) ->
# Bibliogram doesn't have a 1:1 matching to Instagram URLs for users,
# so a "/u" is appended if the requested path doesn't explicitly include
# "/p" for a post or an empty path for the home page.
if String.length(path) > 0 and
!String.starts_with?(path, "p/") and
!String.starts_with?(path, "u/") do
"#{instance}/u"
else
instance
end
String.match?(service, @fandom_regex) -> String.match?(service, @fandom_regex) ->
# Fandom links require the subdomain to be preserved, otherwise the # Fandom links require the subdomain to be preserved, otherwise the
# requested path won't work. # requested path won't work.

10
lib/farside/router.ex
View file

@ -56,16 +56,13 @@ defmodule Farside.Router do
Enum.join(glob, "/") Enum.join(glob, "/")
end end
instance = cond do cond do
conn.assigns[:throttle] != nil -> conn.assigns[:throttle] != nil ->
Farside.get_service(service_name) send_resp(conn, :too_many_requests, "Too many requests - max request rate is 1 per second")
|> Farside.last_instance
|> Farside.amend_instance(service_name, path)
true -> true ->
Farside.get_service(service_name) instance = Farside.get_service(service_name)
|> Farside.pick_instance |> Farside.pick_instance
|> Farside.amend_instance(service_name, path) |> Farside.amend_instance(service_name, path)
end
# Redirect to the available instance # Redirect to the available instance
conn conn
@ -76,3 +73,4 @@ defmodule Farside.Router do
) )
end end
end end
end

45
services-full.json
View file

@ -2,19 +2,31 @@
{ {
"type": "libreddit", "type": "libreddit",
"test_url": "/r/popular", "test_url": "/r/popular",
"fallback": "https://libreddit.spike.codes", "fallback": "https://redlib.freedit.eu",
"instances": [ "instances": [
"https://l.opnxng.com", "https://l.opnxng.com",
"https://libreddit.freedit.eu", "https://libreddit.bus-hit.me",
"https://libreddit.kylrth.com",
"https://libreddit.lunar.icu",
"https://libreddit.northboot.xyz",
"https://libreddit.privacydev.net",
"https://libreddit.projectsegfau.lt", "https://libreddit.projectsegfau.lt",
"https://libreddit.strongthany.cc", "https://reddit.idevicehacked.com",
"https://lr.artemislena.eu",
"https://reddit.invak.id", "https://reddit.invak.id",
"https://reddit.simo.sh" "https://redlib.catsarch.com",
"https://redlib.freedit.eu",
"https://redlib.matthew.science"
]
},
{
"type": "redlib",
"test_url": "/r/popular",
"fallback": "https://redlib.freedit.eu",
"instances": [
"https://l.opnxng.com",
"https://libreddit.bus-hit.me",
"https://libreddit.projectsegfau.lt",
"https://reddit.idevicehacked.com",
"https://reddit.invak.id",
"https://redlib.catsarch.com",
"https://redlib.freedit.eu",
"https://redlib.matthew.science"
] ]
}, },
{ {
@ -53,14 +65,17 @@
] ]
}, },
{ {
"type": "bibliogram", "type": "proxigram",
"test_url": "/p/Ch2WRmiLMjj", "test_url": "/p/Ch2WRmiLMjj",
"fallback": "https://bibliogram.1d4.us", "fallback": "https://ig.opnxng.com",
"instances": [ "instances": [
"https://bibliogram.1d4.us", "https://ig.opnxng.com",
"https://bibliogram.froth.zone", "https://proxigram.lunar.icu",
"https://bibliogram.priv.pw", "https://gram.whatever.social",
"https://ig.femboy.hu" "https://ig.snine.nl",
"https://ig.floppa.one",
"https://proxigram.kyun.li",
"https://proxigram.ducks.party"
] ]
}, },
{ {

48
services.json
View file

@ -2,18 +2,29 @@
{ {
"type": "libreddit", "type": "libreddit",
"test_url": "/r/popular", "test_url": "/r/popular",
"fallback": "https://libreddit.spike.codes", "fallback": "https://redlib.freedit.eu",
"instances": [ "instances": [
"https://l.opnxng.com", "https://l.opnxng.com",
"https://libreddit.kylrth.com", "https://libreddit.bus-hit.me",
"https://libreddit.lunar.icu",
"https://libreddit.northboot.xyz",
"https://libreddit.privacydev.net",
"https://libreddit.projectsegfau.lt", "https://libreddit.projectsegfau.lt",
"https://libreddit.strongthany.cc", "https://reddit.idevicehacked.com",
"https://lr.artemislena.eu",
"https://reddit.invak.id", "https://reddit.invak.id",
"https://reddit.simo.sh" "https://redlib.catsarch.com",
"https://redlib.matthew.science"
]
},
{
"type": "redlib",
"test_url": "/r/popular",
"fallback": "https://redlib.freedit.eu",
"instances": [
"https://l.opnxng.com",
"https://libreddit.bus-hit.me",
"https://libreddit.projectsegfau.lt",
"https://reddit.idevicehacked.com",
"https://reddit.invak.id",
"https://redlib.catsarch.com",
"https://redlib.matthew.science"
] ]
}, },
{ {
@ -47,14 +58,17 @@
] ]
}, },
{ {
"type": "bibliogram", "type": "proxigram",
"test_url": "/p/Ch2WRmiLMjj", "test_url": "/p/Ch2WRmiLMjj",
"fallback": "https://bibliogram.1d4.us", "fallback": "https://ig.opnxng.com",
"instances": [ "instances": [
"https://bibliogram.1d4.us", "https://ig.opnxng.com",
"https://bibliogram.froth.zone", "https://proxigram.lunar.icu",
"https://bibliogram.priv.pw", "https://gram.whatever.social",
"https://ig.femboy.hu" "https://ig.snine.nl",
"https://ig.floppa.one",
"https://proxigram.kyun.li",
"https://proxigram.ducks.party"
] ]
}, },
{ {
@ -125,6 +139,9 @@
"https://nitter.io.lol", "https://nitter.io.lol",
"https://nitter.ktachibana.party", "https://nitter.ktachibana.party",
"https://nitter.kylrth.com", "https://nitter.kylrth.com",
"https://nitter.kavin.rocks",
"https://nitter.unixfox.eu",
"https://n.actionsack.com",
"https://nitter.moomoo.me", "https://nitter.moomoo.me",
"https://nitter.net", "https://nitter.net",
"https://nitter.no-logs.com", "https://nitter.no-logs.com",
@ -188,7 +205,8 @@
"https://translate.dr460nf1r3.org", "https://translate.dr460nf1r3.org",
"https://translate.whateveritworks.org", "https://translate.whateveritworks.org",
"https://translate.sapti.me", "https://translate.sapti.me",
"https://translate.colinslegacy.com" "https://translate.colinslegacy.com",
"https://translate.igna.rocks"
] ]
}, },
{ {