From a256caebedd9efac708c82b6296d090d06623414 Mon Sep 17 00:00:00 2001 From: simon987 Date: Tue, 26 May 2020 20:49:09 -0400 Subject: [PATCH] initial commit --- .gitmodules | 3 + docker_mysql | 15 ++++ docker_remove_exited | 3 + git_filterbranch | 18 +++++ heic2jpg | 3 + img_optim | 6 ++ img_optim_lossy | 4 ++ imhash | Bin 0 -> 14424 bytes inb4404 | 144 +++++++++++++++++++++++++++++++++++++ kill_nonessential_services | 14 ++++ lddpath | 3 + lib/argparse | 81 +++++++++++++++++++++ sc_layout | 3 + sc_update | 5 ++ scripts-private | 1 + tgz2xz | 9 +++ webmenc | 21 ++++++ windows/Caps_to_escape.reg | 3 + windows/Win10Repeat.reg | 3 + xz_enc | 5 ++ ytdl | 3 + zip2xz | 9 +++ 22 files changed, 356 insertions(+) create mode 100644 .gitmodules create mode 100755 docker_mysql create mode 100755 docker_remove_exited create mode 100755 git_filterbranch create mode 100755 heic2jpg create mode 100755 img_optim create mode 100755 img_optim_lossy create mode 100755 imhash create mode 100755 inb4404 create mode 100755 kill_nonessential_services create mode 100755 lddpath create mode 100755 lib/argparse create mode 100755 sc_layout create mode 100755 sc_update create mode 160000 scripts-private create mode 100755 tgz2xz create mode 100755 webmenc create mode 100644 windows/Caps_to_escape.reg create mode 100644 windows/Win10Repeat.reg create mode 100755 xz_enc create mode 100755 ytdl create mode 100755 zip2xz diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e6cca85 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "scripts-private"] + path = scripts-private + url = ssh://git@raw.simon987.net:222/simon987/scripts-private.git diff --git a/docker_mysql b/docker_mysql new file mode 100755 index 0000000..dc3daa1 --- /dev/null +++ b/docker_mysql @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +docker run -d --name tmp_mysql1 \ + -e MYSQL_ROOT_PASSWORD="rootpw" \ + -e MYSQL_ROOT_HOST="%" \ + mysql/mysql-server:5.7 + +sleep 30 + +containerip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tmp_mysql1) + +echo "Login with 'mysql -h ${containerip} -uroot -prootpw'" + +read _ +docker rm -f tmp_mysql1 diff --git a/docker_remove_exited b/docker_remove_exited new file mode 100755 index 0000000..40ed24c --- /dev/null +++ b/docker_remove_exited @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker ps -a | grep Exit | cut -d ' ' -f 1 | xargs docker rm diff --git a/git_filterbranch b/git_filterbranch new file mode 100755 index 0000000..1eac8a5 --- /dev/null +++ b/git_filterbranch @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +git filter-branch --env-filter ' +WRONG_EMAIL="wrong@example.com" +NEW_NAME="New Name Value" +NEW_EMAIL="correct@example.com" + +if [ "$GIT_COMMITTER_EMAIL" = "$WRONG_EMAIL" ] +then + export GIT_COMMITTER_NAME="$NEW_NAME" + export GIT_COMMITTER_EMAIL="$NEW_EMAIL" +fi +if [ "$GIT_AUTHOR_EMAIL" = "$WRONG_EMAIL" ] +then + export GIT_AUTHOR_NAME="$NEW_NAME" + export GIT_AUTHOR_EMAIL="$NEW_EMAIL" +fi +' --tag-name-filter cat -- --branches --tags diff --git a/heic2jpg b/heic2jpg new file mode 100755 index 0000000..779d3bc --- /dev/null +++ b/heic2jpg @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +find "$1" -iname "*.heic" -exec heif-convert {} {}.jpg \; diff --git a/img_optim b/img_optim new file mode 100755 index 0000000..44eb06c --- /dev/null +++ b/img_optim @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +find "$1" -iname "*.jpeg" -exec jpegoptim {} \; +find "$1" -iname "*.jpg" -exec jpegoptim {} \; + +# TODO optipng diff --git a/img_optim_lossy b/img_optim_lossy new file mode 100755 index 0000000..9436f38 --- /dev/null +++ b/img_optim_lossy @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +find "$1" -iname "*.jpeg" -exec jpegoptim -m85 {} \; +find "$1" -iname "*.jpg" -exec jpegoptim -m85 {} \; diff --git a/imhash b/imhash new file mode 100755 index 0000000000000000000000000000000000000000..5be28bb1b58271fc9ab19a48015850d5af237232 GIT binary patch literal 14424 zcmeHOe{37o9e+t$>hy=5u@ZJNmnlzBD9SPZ4zP{+E}ZhN!y@;5t^zLv>yerE*nEc{1|pA(5-|Y9rX73zI$Ki za;x{(m^A5LbocrGe(!zX-Mc%#mv6P`TS`kx1eY?gSr9k9)RY92V!vMzfPknMi*Q~c zmWvC(&yzA!A20!0v-FhCMVFG70m<(tN^i$YlpQ0tkdXX}m2p)8qo^<~$!|W9inrtU zs3RzhtS{8(m1-eu=u+)xwE9uVV>PDVv4*+$nDU!Y$bQ6<-<0y3Qhtnr?GFUEhkIkUM@ib**AFR0QSI*w$KoOsjfB%-C^}Lp zQw~u)BDU(SjZLAozO}yf1-eeu_9XhEwW%1AP}`i?74J=iBdOXsDWnl8mBT`Jn|aXL zEGAzf=#TEhQgIN-^TxpPSULHukmDu9(pYQ>MZ27gQFwmk`S3VhXeh_k{K@hmB_9k( zwO~2(Q&*dEngbS?q&!bs-bx8`m7P*-_ z)2`YgpUt5VYb_=sy# z)@rPO6=Gx6>o|K>20#(tL%RI5fwJl^#M3KF{7@Jw@61x3zoJ*Q=F zakcmjNaPP8dNy*jqQ(EVmaRF7*EWdd%buQH|C-5?8xkb+v9iaB)-vzvnWJYO_S`>l zi5Qvij9iBJQUkG``M{!SwP>_#!vhFt(QcZn9hs}8*IzyJjFw%Wfq+$oe6= z4hyNDl$Ms%cghN$%iT^DTwJVzJ=O~TMvW{ow4L_&XYm<8L%W>fACR89nXkv`?zdhR zUt(X%(x5c8X6X-wES>wOElYRHN)G3K2}OQ0JagMYb@_Ps$7z1^zulU7{uV9ssy6)k z>1}O#cK%)rl$KrMk1CK2k3vUNW4STySviE( zi%EsB-ze?ZuO(Q0PRpFuo_^GQ-Zd-4z7sAn5~cDWG28(MKyx3^|yf^EUf$L*N? zegN_H`sz>UnUQs(iQV{&!MyM0Tm z-m=5z^NE@ox#R?ti&a3mQU#RjPrwu{BGuGLN3kkZ-hbt))IuV{OhhEn6AmXyyo<$s zl2F}hPhp+QJ7>Abiv_7wopaqLI^t4Yx8uGr~z2_0k_#}NEy@R=$a-4*wi zHo7bCDQk3lv-29=)pySiy4UQzAn2|iUa-}D`hv3KB@5m4P3|>8cQtHa3!g@}t7#F1 z#c>43PSitdv0@9wYo&NC6t9KiHB&szj2DQz>k;1;ORlHUXYfA6v91GANT)a%a5CUz zz{!A<0Ve}a2Am8y8E`W2|B(US@5cMv@QN>s9SkxWnR=Ec-4p@pFpj zeReApFV{UmFIPP8x1(<=Dy*M5XCx#~-$E$-?NO-+-g`z%dQ{{GAFy4ukG_GZRGJ{v zHa^9fy9p%|Fh#TgO+_yL0r~rn7Luu4Y=YRQ1S~(M8p7?KPvD`&Obg=B_CIMtei+*bywB$zlXxUa$D6K)TX>G3U5%@rSLX|dli0P;lm1l zrSPc26AHPS!GK9^ZECvGTOCYx_IP#L*zMiuTjyI-<6mtOLikcWv~M-sA$-`x8cp^K zUp$eH`nux-zK($yHiX9_N)&8ttx1Qwh1AjRSmBH8j;D6_nV3$R8rn)5OT_0eA?T9P z-Y_{Rs=qf)RmN~mM+b4nj@cM=3AroQ7sVdlWVkOH>WRRYInyH)PA0>dxHJ#|4%@_ zk66n*D+BrqIIR9T6^PMmv$y-#LH|DP5w72*ZWP8f%Ae;EzF#@)W)PGE_UCzpaawhl zZbu^7pYQK2kWrQF&+`-`&r2|o#h$-wA*DHh{rSFUtX6*9ezs%W4n55&%=5g$$o;ps zUxgK42SGVyf1cMECzK)AZ|^@{hUDL?00Pb{2XND=Of$O+dr=SIsd||EEWD# zu?(C)=J$i3{?d;JeXWA)E@O;biMDJ{ib|D(`h&yc(a$c&%+{C_)k zd*&IRfRD%G&+i4tYbe_6^ 6: + thread_tmp = thread_link.split('/')[6].split('#')[0] + + if args.use_names or os.path.exists(os.path.join(workpath, 'downloads', board, thread_tmp)): + thread = thread_tmp + + directory = os.path.join(workpath, 'downloads', board, thread) + if not os.path.exists(directory): + os.makedirs(directory) + + while True: + try: + regex = '(\/\/i(?:s|)\d*\.(?:4cdn|4chan)\.org\/\w+\/(\d+\.(?:jpg|png|gif|webm)))' + regex_result = list(set(re.findall(regex, load(thread_link).decode('utf-8')))) + regex_result = sorted(regex_result, key=lambda tup: tup[1]) + regex_result_len = len(regex_result) + regex_result_cnt = 1 + + for link, img in regex_result: + img_path = os.path.join(directory, img) + if not os.path.exists(img_path): + data = load('https:' + link) + + output_text = board + '/' + thread + '/' + img + if args.with_counter: + output_text = '[' + str(regex_result_cnt).rjust(len(str(regex_result_len))) + '/' + str(regex_result_len) + '] ' + output_text + + log.info(output_text) + + with open(img_path, 'wb') as f: + f.write(data) + + ################################################################################## + # saves new images to a seperate directory + # if you delete them there, they are not downloaded again + # if you delete an image in the 'downloads' directory, it will be downloaded again + copy_directory = os.path.join(workpath, 'new', board, thread) + if not os.path.exists(copy_directory): + os.makedirs(copy_directory) + copy_path = os.path.join(copy_directory, img) + with open(copy_path, 'wb') as f: + f.write(data) + ################################################################################## + regex_result_cnt += 1 + + except urllib.error.HTTPError as err: + time.sleep(10) + try: + load(thread_link) + except urllib.error.HTTPError as err: + log.info('%s 404\'d', thread_link) + break + continue + except (urllib.error.URLError, http.client.BadStatusLine, http.client.IncompleteRead): + if not args.less: + log.warning('Something went wrong') + + if not args.less: + log.info('Checking ' + board + '/' + thread) + time.sleep(20) + +def download_from_file(filename): + running_links = [] + while True: + processes = [] + for link in [_f for _f in [line.strip() for line in open(filename) if line[:4] == 'http'] if _f]: + if link not in running_links: + running_links.append(link) + log.info('Added ' + link) + + process = Process(target=download_thread, args=(link, )) + process.start() + processes.append([process, link]) + + if len(processes) == 0: + log.warning(filename + ' empty') + + if args.reload: + time.sleep(60 * 5) # 5 minutes + links_to_remove = [] + for process, link in processes: + if not process.is_alive(): + links_to_remove.append(link) + else: + process.terminate() + + for link in links_to_remove: + for line in fileinput.input(filename, inplace=True): + print(line.replace(link, '-' + link), end='') + running_links.remove(link) + log.info('Removed ' + link) + if not args.less: + log.info('Reloading ' + args.thread[0]) # thread = filename here; reloading on next loop + else: + break + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + pass + diff --git a/kill_nonessential_services b/kill_nonessential_services new file mode 100755 index 0000000..ead2e29 --- /dev/null +++ b/kill_nonessential_services @@ -0,0 +1,14 @@ +#!/bin/bash + +systemctl stop docker +systemctl stop mongod +systemctl stop mongodb +systemctl stop redis-server +systemctl stop redis +systemctl stop postgresql +systemctl stop nginx +systemctl stop vsftpd +killall nextcloud +killall cmus + + diff --git a/lddpath b/lddpath new file mode 100755 index 0000000..48209e5 --- /dev/null +++ b/lddpath @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +ldd "$1" | grep so | sed -e '/^[^\t]/ d' | sed -e 's/\t//' | sed -e 's/.*=..//' | sed -e 's/ (0.*)//' diff --git a/lib/argparse b/lib/argparse new file mode 100755 index 0000000..25f935e --- /dev/null +++ b/lib/argparse @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# Use python's argparse module in shell scripts +# +# The function `argparse` parses its arguments using +# argparse.ArgumentParser; the parser is defined in the function's +# stdin. +# +# Executing ``argparse.bash`` (as opposed to sourcing it) prints a +# script template. +# +# https://github.com/nhoffman/argparse-bash +# MIT License - Copyright (c) 2015 Noah Hoffman + +argparse(){ + argparser=$(mktemp 2>/dev/null || mktemp -t argparser) + cat > "$argparser" <> "$argparser" + + cat >> "$argparser" < /dev/null; then + eval $(python "$argparser" "$@") + retval=0 + else + python "$argparser" "$@" + retval=1 + fi + + rm "$argparser" + return $retval +} + +# print a script template when this script is executed +if [[ $0 == *argparse.bash ]]; then + cat <=1080][fps>30]/bestvideo[vcodec=vp9.2][height>=1080][fps>30]/bestvideo[vcodec=vp9][height>=1080][fps>30]/bestvideo[vcodec^=av01][height>=1080]/bestvideo[vcodec=vp9.2][height>=1080]/bestvideo[vcodec=vp9][height>=1080]/bestvideo[height>=1080]/bestvideo[vcodec^=av01][height>=720][fps>30]/bestvideo[vcodec=vp9.2][height>=720][fps>30]/bestvideo[vcodec=vp9][height>=720][fps>30]/bestvideo[vcodec^=av01][height>=720]/bestvideo[vcodec=vp9.2][height>=720]/bestvideo[vcodec=vp9][height>=720]/bestvideo[height>=720]/bestvideo)+(bestaudio[acodec=opus]/bestaudio)/best" --force-ipv4 --ignore-errors --no-continue --no-overwrites --add-metadata --write-info-json --write-annotations --write-thumbnail "$1" diff --git a/zip2xz b/zip2xz new file mode 100755 index 0000000..83910ae --- /dev/null +++ b/zip2xz @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +XZ_OPT=-9 + +F=$(basename "$1") + +mkdir /tmp/"$F" +unzip "$1" -d /tmp/"$1" +tar -cJf "$1".tar.xz /tmp/"$F"