#!/bin/sh

#
# script extracts youtube links from a webpage and 
# creates a file from them in the current directory
#
# !!! needs lynx installed !!!

#### Variablen #########################################################

#set -xv;

LAST_WEB_PAGE_NUMBER="69"; 	# last thread page from: 03.11.2024

# see 
WEBPAGE_URL="https://www.kodinerds.net/thread/13225-musik-tipps";

# what kind of links are extracted 
SEARCH_FOR='youtube|youtu.be|spotify';

# output file name
LINK_LIST="youtube_links.txt";

LYNX_INSTALLED=$(which lynx);
[ -z ${LYNX_INSTALLED} ] && echo -e "\n ERROR: lynx not installed !!!\n" && exit 13;

########################################################################

# clean up
[ -e "${LINK_LIST}" ] && rm -rf ${LINK_LIST};


for i in $(seq 1 ${LAST_WEB_PAGE_NUMBER}); do
 	
	lynx -dump -listonly -nonumbers ${WEBPAGE_URL}/?pageNo=$i |sort -u | grep -iE ${SEARCH_FOR} >> ${LINK_LIST};
	 
done

# delete spaces and empty lines
	sed -i -e 's/^ //g' -e 's/ $//g' -e '/^ *$/d' ${LINK_LIST};

exit 0;

