From e1e46dcd9271e68492db5fe259c1593f02e7ef84 Mon Sep 17 00:00:00 2001
From: Ben Busby <contact@benbusby.com>
Date: Thu, 4 Aug 2022 13:16:58 -0600
Subject: [PATCH] Remove cloudflare filter from update workflow
The cloudflare filter, when performed as part of the github action
workflow, doesn't seem to work nearly as reliably when run on an actual
machine.
The farside server will instead run the un-cloudflare script whenever it
pulls in new changes to services-full.json, which should be a much more
reliable approach to filtering out cloudflare instances.
---
.github/workflows/update-instances.yml | 32 -----------
services.json | 73 +++++---------------------
tools/un-cloudflare.sh | 4 +-
3 files changed, 14 insertions(+), 95 deletions(-)
diff --git a/.github/workflows/update-instances.yml b/.github/workflows/update-instances.yml
index 17da660..7f483ac 100644
--- a/.github/workflows/update-instances.yml
+++ b/.github/workflows/update-instances.yml
@@ -27,33 +27,6 @@ jobs:
sed -i 's/\/"/"/g' services-full.json
}
- function remove_cf_instances() {
- rm -f out.json
- file="services-full.json"
-
- while read -r line; do
- if [[ "$line" == "\"https://"* ]]; then
- domain=$(echo "$line" | sed -e "s/^\"https:\/\///" -e "s/\",//" -e "s/\"//")
- ns=$(dig ns "$domain" || true)
- if [[ "$ns" == *"cloudflare"* ]]; then
- echo "\"$domain\" using cloudflare, skipping..."
- elif [[ "$ns" != *"NOERROR"* ]]; then
- echo "Unable to verify records for \"$domain\", skipping..."
- else
- echo "$line" >> out.json
- fi
- else
- echo "$line" >> out.json
- fi
- done <$file
-
- # Remove any trailing commas from new instance lists
- sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
-
- cat out.json | jq --indent 2 . > services.json
- rm -f out.json
- }
-
# ==============================================================
# Git config
# ==============================================================
@@ -216,11 +189,6 @@ jobs:
# TODO: Update instances for other services
# ==============================================================
- # ==============================================================
- # Remove cloudflare instances
- # ==============================================================
- remove_cf_instances
-
# ==============================================================
# Push changes
# ==============================================================
diff --git a/services.json b/services.json
index 4300fdf..ea5ce7d 100644
--- a/services.json
+++ b/services.json
@@ -4,29 +4,19 @@
"test_url": "/r/popular",
"fallback": "https://libredd.it",
"instances": [
- "https://libreddit.albony.xyz",
"https://libreddit.spike.codes",
"https://libreddit.dothq.co",
- "https://libreddit.kavin.rocks",
"https://libreddit.40two.app",
- "https://reddit.invak.id",
- "https://lr.riverside.rocks",
- "https://libreddit.silkky.cloud",
- "https://libreddit.database.red",
- "https://libreddit.exonip.de",
- "https://libreddit.domain.glass",
- "https://libreddit.trevorthalacker.com",
+ "https://reddit.phii.me",
+ "https://libreddit.sugoma.tk",
"https://reddit.artemislena.eu",
"https://libreddit.some-things.org",
"https://reddit.stuehieyr.com",
- "https://lr.mint.lgbt",
"https://libreddit.igna.rocks",
"https://libreddit.autarkic.org",
"https://libreddit.flux.industries",
- "https://libreddit.drivet.xyz",
"https://libreddit.de",
- "https://libreddit.bus-hit.me",
- "https://de.leddit.xyz"
+ "https://libreddit.bus-hit.me"
]
},
{
@@ -37,18 +27,15 @@
"https://incogsnoo.com",
"https://rdt.trom.tf",
"https://reddit.lol",
+ "https://snoo.ioens.is",
"https://teddit.adminforge.de",
"https://teddit.bus-hit.me",
- "https://teddit.domain.glass",
- "https://teddit.encrypted-data.xyz",
"https://teddit.froth.zone",
- "https://teddit.garudalinux.org",
- "https://teddit.ggc-project.de",
- "https://teddit.httpjames.space",
- "https://teddit.namazso.eu",
+ "https://teddit.nautolan.racing",
"https://teddit.net",
"https://teddit.pussthecat.org",
"https://teddit.sethforprivacy.com",
+ "https://teddit.tinfoil-hat.net",
"https://teddit.totaldarkness.net",
"https://teddit.zaggy.nl"
]
@@ -58,7 +45,6 @@
"test_url": "/u/officialrickastley",
"fallback": "https://bibliogram.art",
"instances": [
- "https://bib.actionsack.com",
"https://bibliogram.1d4.us",
"https://bibliogram.art",
"https://bibliogram.froth.zone",
@@ -67,8 +53,6 @@
"https://bibliogram.pussthecat.org",
"https://bibliogram.snopyta.org",
"https://ig.beparanoid.de",
- "https://ig.funami.tech",
- "https://ig.tokhmi.xyz",
"https://insta.trom.tf",
"https://insta.tromdienste.de"
]
@@ -79,13 +63,9 @@
"fallback": "https://invidious.snopyta.org",
"instances": [
"https://inv.bp.projectsegfau.lt",
- "https://inv.privacy.com.de",
- "https://inv.riverside.rocks",
"https://inv.vern.cc",
"https://invidious.esmailelbob.xyz",
"https://invidious.flokinet.to",
- "https://invidious.kavin.rocks",
- "https://invidious.namazso.eu",
"https://invidious.nerdvpn.de",
"https://invidious.osi.kr",
"https://invidious.projectsegfau.lt",
@@ -106,10 +86,6 @@
"test_url": "/watch?v=eBGIQ7ZuuiU",
"fallback": "https://piped.kavin.rocks",
"instances": [
- "https://piped.kavin.rocks",
- "https://piped.silkky.cloud",
- "https://piped.tokhmi.xyz",
- "https://piped.moomoo.me",
"https://piped.mint.lgbt"
]
},
@@ -124,29 +100,21 @@
"https://nitter.nixnet.services",
"https://nitter.fdn.fr",
"https://nitter.1d4.us",
- "https://nitter.kavin.rocks",
"https://nitter.vxempire.xyz",
"https://nitter.unixfox.eu",
"https://nitter.eu",
- "https://nitter.namazso.eu",
"https://nitter.mailstation.de",
- "https://n.actionsack.com",
"https://nitter.cattube.org",
- "https://nitter.exonip.de",
"https://twitr.gq",
- "https://nitter.moomoo.me",
"https://bird.trom.tf",
"https://nitter.it",
- "https://twitter.censors.us",
"https://nitter.grimneko.de",
- "https://nitter.koyu.space",
"https://n.0x0.st",
"https://n.hyperborea.cloud",
"https://nitter.ca",
"https://twitter.076.ne.jp",
"https://nitter.sethforprivacy.com",
"https://nitter.bus-hit.me",
- "https://de.nttr.stream",
"https://n.l5.ca",
"https://unofficialbird.com"
]
@@ -177,10 +145,9 @@
"https://tl.vern.cc",
"https://translate.beparanoid.de",
"https://translate.bus-hit.me",
- "https://translate.namazso.eu",
+ "https://translate.josias.dev",
"https://translate.northboot.xyz",
"https://translate.priv.pw",
- "https://translate.riverside.rocks",
"https://translate.slipfox.xyz",
"https://translate.tiekoetter.com"
]
@@ -202,7 +169,6 @@
"instances": [
"https://i.bcow.xyz",
"https://rimgo.pussthecat.org",
- "https://img.riverside.rocks",
"https://rimgo.totaldarkness.net",
"https://rimgo.bus-hit.me"
]
@@ -213,15 +179,10 @@
"fallback": "https://whoogle.fossho.st",
"instances": [
"https://gowogle.voring.me",
- "https://s.tokhmi.xyz",
- "https://search.albony.xyz",
- "https://search.dr460nf1r3.org",
- "https://search.garudalinux.org",
"https://search.sethforprivacy.com",
- "https://search.wef.lol",
- "https://whoogle.dcs0.hu",
"https://whoogle.esmailelbob.xyz",
"https://whoogle.fossho.st",
+ "https://whoogle.lunar.icu",
"https://whoogle.privacydev.net",
"https://www.whooglesearch.ml"
]
@@ -235,7 +196,6 @@
"https://dynabyte.ca",
"https://search.neet.works",
"https://searx.divided-by-zero.eu",
- "https://searx.gnu.style",
"https://searx.ru",
"https://searx.sp-codes.de",
"https://searx.stuehieyr.com",
@@ -249,10 +209,10 @@
"test_url": "/search?q=<%=query%>",
"fallback": "https://searx.be",
"instances": [
+ "https://darmarit.org/searx",
"https://de.xcxc.ml",
"https://jackgoss.xyz",
"https://priv.au",
- "https://s.frlt.one",
"https://s.zhaocloud.net",
"https://saber.tk",
"https://search.0relay.com",
@@ -263,12 +223,8 @@
"https://search.mdosch.de",
"https://search.neet.works",
"https://search.ononoki.org",
- "https://search.privacyguides.net",
"https://search.projectsegfau.lt",
- "https://search.rabbit-company.com",
"https://search.sapti.me",
- "https://search.teamriverbubbles.com",
- "https://search.unlocked.link",
"https://search.zzls.xyz",
"https://searx.be",
"https://searx.becomesovran.com",
@@ -278,14 +234,12 @@
"https://searx.fmac.xyz",
"https://searx.mha.fi",
"https://searx.mistli.net",
- "https://searx.namejeff.xyz",
"https://searx.priv.pw",
"https://searx.prvcy.eu",
"https://searx.sev.monster",
+ "https://searx.slipfox.xyz/searx",
"https://searx.tiekoetter.com",
- "https://searx.zcyph.cc",
"https://searxng.tordenskjold.de",
- "https://searxng.zackptg5.com",
"https://serx.ml",
"https://swag.pw",
"https://xcxc.ml"
@@ -296,11 +250,10 @@
"test_url": "/wiki/Wikipedia?lang=en",
"fallback": "https://wikiless.org",
"instances": [
- "https://wiki.604kph.xyz",
"https://wiki.froth.zone",
"https://wiki.slipfox.xyz",
"https://wikiless.esmailelbob.xyz",
- "https://wikiless.funami.tech",
+ "https://wikiless.lunar.icu",
"https://wikiless.northboot.xyz",
"https://wikiless.org",
"https://wikiless.sethforprivacy.com",
@@ -336,9 +289,7 @@
"fallback": "https://librex.beparanoid.de",
"instances": [
"https://librex.beparanoid.de",
- "https://search.davidovski.xyz",
- "https://search.funami.tech",
- "https://librex.catalyst.sx"
+ "https://search.davidovski.xyz"
]
},
{
diff --git a/tools/un-cloudflare.sh b/tools/un-cloudflare.sh
index b10178b..dd5c716 100755
--- a/tools/un-cloudflare.sh
+++ b/tools/un-cloudflare.sh
@@ -10,7 +10,7 @@ while read -r line; do
ns=$(dig ns "$domain" || true)
if [[ "$ns" == *"cloudflare"* ]]; then
echo "\"$domain\" using cloudflare, skipping..."
- elif [ ${#ns} -eq 0 ]; then
+ elif [[ "$ns" != *"NOERROR"* ]]; then
echo "Unable to verify records for \"$domain\", skipping..."
else
echo "$line" >> out.json
@@ -21,7 +21,7 @@ while read -r line; do
done <$file
# Remove any trailing commas from new instance lists
-sed -i '' -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
+sed -i -e ':begin' -e '$!N' -e 's/,\n]/\n]/g' -e 'tbegin' -e 'P' -e 'D' out.json
cat out.json | jq --indent 2 . > services.json
rm -f out.json