shell commands to gather data about blocked instances across multiple lemmy servers

# create an empty directory to work in to avoid clutter
# suggest running this to create a directory for today's date:
DATE=`date +%Y-%m-%d`; mkdir $DATE; cd $DATE

# list of top 10 lemmy servers in terms of users from
# https://fedidb.com/software/lemmy
LEMMY_SERVERS="lemmy.blahaj.zone lemmy.ca lemmy.dbzer0.com lemmy.ml lemmy.one lemmy.world lemmy.zip programming.dev sh.itjust.works ttrpg.network"

# loop through list of lemmy servers to download json files of blocked instances and extract the blocked domains to a txt file
# also get censured sites reported from a lemmy server to fediseer and extract domains to a txt file
# Lemmy REST API: https://lemmy.readme.io/
# Fediseer REST API: https://fediseer.com/api
for site in `echo $LEMMY_SERVERS`
do
  echo $site
  site_base_fname=`echo $site | tr \. _`

  # just get lemmy site stats in a JSON file
  curl -s --request GET \
     --url https://$site/api/v3/site \
     --header 'accept: application/json' > site_${site_base_fname}.json

  curl -s --request GET \
     --url https://$site/api/v3/federated_instances?kind=blocked \
     --header 'accept: application/json' > blocked_${site_base_fname}.json
  cat blocked_${site_base_fname}.json | jq -r '.federated_instances.blocked.[].domain' | sort -u > blocked_${site_base_fname}.txt

  curl -s --request GET \
      --url https://fediseer.com/api/v1/censures_given/$site \
      --header 'accept: application/json' >censured_${site_base_fname}.json
  cat censured_${site_base_fname}.json | jq -r '.instances.[].domain' | sort -u > censured_${site_base_fname}.txt
done

# instances blocked on sjw but not censured by sjw on fediseer
comm -23 blocked_sh_itjust_works.txt censured_sh_itjust_works.txt

# instances censured by sjw on fediseer but not blocked
comm -13 blocked_sh_itjust_works.txt censured_sh_itjust_works.txt

# number of common blocked instances for both lemmy.ca and sh.itjust.works
comm -12 blocked_lemmy_ca.txt blocked_sh_itjust_works.txt | wc -l

# instances only blocked on lemmy.ca but not on sh.itjust.works
comm -23 blocked_lemmy_ca.txt blocked_sh_itjust_works.txt

# instances only blocked on sh.itjust.works but not on lemmy.ca
comm -13 blocked_lemmy_ca.txt blocked_sh_itjust_works.txt

# instances blocked across at least two lemmy servers
# h/t DDG Search Assist AI
awk '{instances[$0]++} END { for (instance in instances) {if(instances[instance]>1) {print instances[instance], instance }}}' blocked*.txt | sort -k1,1nr -k2,2

# instances blocked on at least one lemmy server
awk '{instances[$0]++} END { for (instance in instances) { print instances[instance], instance }}' blocked*.txt | sort -k1,1nr -k2,2

# instances blocked across at least two lemmy servers but not sh.itjust.works
awk '{instances[$0]++} END { for (instance in instances) {if(instances[instance]>1) {print instance }}}' blocked*.txt | sort > peer_blocked_by_at_least_2.txt
comm -13 blocked_sh_itjust_works.txt peer_blocked_by_at_least_2.txt