commit a256caebedd9efac708c82b6296d090d06623414 Author: simon987 Date: Tue May 26 20:49:09 2020 -0400 initial commit diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..e6cca85 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "scripts-private"] + path = scripts-private + url = ssh://git@raw.simon987.net:222/simon987/scripts-private.git diff --git a/docker_mysql b/docker_mysql new file mode 100755 index 0000000..dc3daa1 --- /dev/null +++ b/docker_mysql @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +docker run -d --name tmp_mysql1 \ + -e MYSQL_ROOT_PASSWORD="rootpw" \ + -e MYSQL_ROOT_HOST="%" \ + mysql/mysql-server:5.7 + +sleep 30 + +containerip=$(docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' tmp_mysql1) + +echo "Login with 'mysql -h ${containerip} -uroot -prootpw'" + +read _ +docker rm -f tmp_mysql1 diff --git a/docker_remove_exited b/docker_remove_exited new file mode 100755 index 0000000..40ed24c --- /dev/null +++ b/docker_remove_exited @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +docker ps -a | grep Exit | cut -d ' ' -f 1 | xargs docker rm diff --git a/git_filterbranch b/git_filterbranch new file mode 100755 index 0000000..1eac8a5 --- /dev/null +++ b/git_filterbranch @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +git filter-branch --env-filter ' +WRONG_EMAIL="wrong@example.com" +NEW_NAME="New Name Value" +NEW_EMAIL="correct@example.com" + +if [ "$GIT_COMMITTER_EMAIL" = "$WRONG_EMAIL" ] +then + export GIT_COMMITTER_NAME="$NEW_NAME" + export GIT_COMMITTER_EMAIL="$NEW_EMAIL" +fi +if [ "$GIT_AUTHOR_EMAIL" = "$WRONG_EMAIL" ] +then + export GIT_AUTHOR_NAME="$NEW_NAME" + export GIT_AUTHOR_EMAIL="$NEW_EMAIL" +fi +' --tag-name-filter cat -- --branches --tags diff --git a/heic2jpg b/heic2jpg new file mode 100755 index 0000000..779d3bc --- /dev/null +++ b/heic2jpg @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +find "$1" -iname "*.heic" -exec heif-convert {} {}.jpg \; diff --git a/img_optim b/img_optim new file mode 100755 index 0000000..44eb06c --- /dev/null +++ b/img_optim @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +find "$1" -iname "*.jpeg" -exec jpegoptim {} \; +find "$1" -iname "*.jpg" -exec jpegoptim {} \; + +# TODO optipng diff --git a/img_optim_lossy b/img_optim_lossy new file mode 100755 index 0000000..9436f38 --- /dev/null +++ b/img_optim_lossy @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + +find "$1" -iname "*.jpeg" -exec jpegoptim -m85 {} \; +find "$1" -iname "*.jpg" -exec jpegoptim -m85 {} \; diff --git a/imhash b/imhash new file mode 100755 index 0000000..5be28bb Binary files /dev/null and b/imhash differ diff --git a/inb4404 b/inb4404 new file mode 100755 index 0000000..dc82fd2 --- /dev/null +++ b/inb4404 @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 + +import urllib.request, urllib.error, urllib.parse, argparse, logging +import os, re, time +import http.client +import fileinput +from multiprocessing import Process + +log = logging.getLogger('inb4404') +workpath = os.path.dirname(os.path.realpath(__file__)) +args = None + +def main(): + global args + parser = argparse.ArgumentParser(description='inb4404') + parser.add_argument('thread', nargs=1, help='url of the thread (or filename; one url per line)') + parser.add_argument('-c', '--with-counter', action='store_true', help='show a counter next the the image that has been downloaded') + parser.add_argument('-d', '--date', action='store_true', help='show date as well') + parser.add_argument('-l', '--less', action='store_true', help='show less information (surpresses checking messages)') + parser.add_argument('-n', '--use-names', default=True, action='store_true', help='use thread names instead of the thread ids (...4chan.org/board/thread/thread-id/thread-name)') + parser.add_argument('-r', '--reload', action='store_true', help='reload the queue file every 5 minutes') + args = parser.parse_args() + + if args.date: + logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%Y-%m-%d %I:%M:%S %p') + else: + logging.basicConfig(level=logging.INFO, format='[%(asctime)s] %(message)s', datefmt='%I:%M:%S %p') + + thread = args.thread[0].strip() + if thread[:4].lower() == 'http': + download_thread(thread) + else: + download_from_file(thread) + +def load(url): + req = urllib.request.Request(url, headers={'User-Agent': '4chan Browser'}) + return urllib.request.urlopen(req).read() + +def download_thread(thread_link): + board = thread_link.split('/')[3] + thread = thread_link.split('/')[5].split('#')[0] + if len(thread_link.split('/')) > 6: + thread_tmp = thread_link.split('/')[6].split('#')[0] + + if args.use_names or os.path.exists(os.path.join(workpath, 'downloads', board, thread_tmp)): + thread = thread_tmp + + directory = os.path.join(workpath, 'downloads', board, thread) + if not os.path.exists(directory): + os.makedirs(directory) + + while True: + try: + regex = '(\/\/i(?:s|)\d*\.(?:4cdn|4chan)\.org\/\w+\/(\d+\.(?:jpg|png|gif|webm)))' + regex_result = list(set(re.findall(regex, load(thread_link).decode('utf-8')))) + regex_result = sorted(regex_result, key=lambda tup: tup[1]) + regex_result_len = len(regex_result) + regex_result_cnt = 1 + + for link, img in regex_result: + img_path = os.path.join(directory, img) + if not os.path.exists(img_path): + data = load('https:' + link) + + output_text = board + '/' + thread + '/' + img + if args.with_counter: + output_text = '[' + str(regex_result_cnt).rjust(len(str(regex_result_len))) + '/' + str(regex_result_len) + '] ' + output_text + + log.info(output_text) + + with open(img_path, 'wb') as f: + f.write(data) + + ################################################################################## + # saves new images to a seperate directory + # if you delete them there, they are not downloaded again + # if you delete an image in the 'downloads' directory, it will be downloaded again + copy_directory = os.path.join(workpath, 'new', board, thread) + if not os.path.exists(copy_directory): + os.makedirs(copy_directory) + copy_path = os.path.join(copy_directory, img) + with open(copy_path, 'wb') as f: + f.write(data) + ################################################################################## + regex_result_cnt += 1 + + except urllib.error.HTTPError as err: + time.sleep(10) + try: + load(thread_link) + except urllib.error.HTTPError as err: + log.info('%s 404\'d', thread_link) + break + continue + except (urllib.error.URLError, http.client.BadStatusLine, http.client.IncompleteRead): + if not args.less: + log.warning('Something went wrong') + + if not args.less: + log.info('Checking ' + board + '/' + thread) + time.sleep(20) + +def download_from_file(filename): + running_links = [] + while True: + processes = [] + for link in [_f for _f in [line.strip() for line in open(filename) if line[:4] == 'http'] if _f]: + if link not in running_links: + running_links.append(link) + log.info('Added ' + link) + + process = Process(target=download_thread, args=(link, )) + process.start() + processes.append([process, link]) + + if len(processes) == 0: + log.warning(filename + ' empty') + + if args.reload: + time.sleep(60 * 5) # 5 minutes + links_to_remove = [] + for process, link in processes: + if not process.is_alive(): + links_to_remove.append(link) + else: + process.terminate() + + for link in links_to_remove: + for line in fileinput.input(filename, inplace=True): + print(line.replace(link, '-' + link), end='') + running_links.remove(link) + log.info('Removed ' + link) + if not args.less: + log.info('Reloading ' + args.thread[0]) # thread = filename here; reloading on next loop + else: + break + + +if __name__ == '__main__': + try: + main() + except KeyboardInterrupt: + pass + diff --git a/kill_nonessential_services b/kill_nonessential_services new file mode 100755 index 0000000..ead2e29 --- /dev/null +++ b/kill_nonessential_services @@ -0,0 +1,14 @@ +#!/bin/bash + +systemctl stop docker +systemctl stop mongod +systemctl stop mongodb +systemctl stop redis-server +systemctl stop redis +systemctl stop postgresql +systemctl stop nginx +systemctl stop vsftpd +killall nextcloud +killall cmus + + diff --git a/lddpath b/lddpath new file mode 100755 index 0000000..48209e5 --- /dev/null +++ b/lddpath @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +ldd "$1" | grep so | sed -e '/^[^\t]/ d' | sed -e 's/\t//' | sed -e 's/.*=..//' | sed -e 's/ (0.*)//' diff --git a/lib/argparse b/lib/argparse new file mode 100755 index 0000000..25f935e --- /dev/null +++ b/lib/argparse @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# Use python's argparse module in shell scripts +# +# The function `argparse` parses its arguments using +# argparse.ArgumentParser; the parser is defined in the function's +# stdin. +# +# Executing ``argparse.bash`` (as opposed to sourcing it) prints a +# script template. +# +# https://github.com/nhoffman/argparse-bash +# MIT License - Copyright (c) 2015 Noah Hoffman + +argparse(){ + argparser=$(mktemp 2>/dev/null || mktemp -t argparser) + cat > "$argparser" <> "$argparser" + + cat >> "$argparser" < /dev/null; then + eval $(python "$argparser" "$@") + retval=0 + else + python "$argparser" "$@" + retval=1 + fi + + rm "$argparser" + return $retval +} + +# print a script template when this script is executed +if [[ $0 == *argparse.bash ]]; then + cat <=1080][fps>30]/bestvideo[vcodec=vp9.2][height>=1080][fps>30]/bestvideo[vcodec=vp9][height>=1080][fps>30]/bestvideo[vcodec^=av01][height>=1080]/bestvideo[vcodec=vp9.2][height>=1080]/bestvideo[vcodec=vp9][height>=1080]/bestvideo[height>=1080]/bestvideo[vcodec^=av01][height>=720][fps>30]/bestvideo[vcodec=vp9.2][height>=720][fps>30]/bestvideo[vcodec=vp9][height>=720][fps>30]/bestvideo[vcodec^=av01][height>=720]/bestvideo[vcodec=vp9.2][height>=720]/bestvideo[vcodec=vp9][height>=720]/bestvideo[height>=720]/bestvideo)+(bestaudio[acodec=opus]/bestaudio)/best" --force-ipv4 --ignore-errors --no-continue --no-overwrites --add-metadata --write-info-json --write-annotations --write-thumbnail "$1" diff --git a/zip2xz b/zip2xz new file mode 100755 index 0000000..83910ae --- /dev/null +++ b/zip2xz @@ -0,0 +1,9 @@ +#!/usr/bin/env bash + +XZ_OPT=-9 + +F=$(basename "$1") + +mkdir /tmp/"$F" +unzip "$1" -d /tmp/"$1" +tar -cJf "$1".tar.xz /tmp/"$F"