From 2d0d281db2898aacd30d241db21a3389c904b2fb Mon Sep 17 00:00:00 2001 From: bel Date: Sun, 15 Oct 2023 10:33:51 -0600 Subject: [PATCH] initial --- .bashrc | 1263 ++++++++++++++++++++++++++++++++++++++++++++++++++++ .gitconfig | 3 + .gitignore | 9 + .vimrc | 64 +++ env.sh | 10 + log.yaml | 44 ++ 6 files changed, 1393 insertions(+) create mode 100644 .bashrc create mode 100644 .gitconfig create mode 100644 .gitignore create mode 100644 .vimrc create mode 100644 env.sh create mode 100644 log.yaml diff --git a/.bashrc b/.bashrc new file mode 100644 index 0000000..9d58d8e --- /dev/null +++ b/.bashrc @@ -0,0 +1,1263 @@ +# INIT +export BASH_SILENCE_DEPRECATION_WARNING=1 +export USER=breel + +#ssh-add ~/.ssh/id_rsa 2> /dev/null > /dev/null & +shopt -s histappend +export HISTSIDE=1000000 +export HISTSIZE=1000000 +export HISTFILESIZE=2000000 +export HISTCONTROL=ignoreboth:erasedups +export HISTTIMEFORMAT="%d/%m/%y %T " +shopt -s checkwinsize +case "$TERM" in + xterm-color|*-256color) color_prompt=yes;; +esac + +login() { + true +} + +if [ -n "$force_color_prompt" ]; then + if [ -x /usr/bin/tput ] && tput setaf 1 >&/dev/null; then + # We have color support; assume it's compliant with Ecma-48 + # (ISO/IEC-6429). (Lack of such support is extremely rare, and such + # a case would tend to support setf rather than setaf.) + color_prompt=yes + else + color_prompt= + fi +fi +if [ "$color_prompt" = yes ]; then + PS1='${debian_chroot:+($debian_chroot)}\[\033[01;32m\]\u@\h\[\033[00m\]:\[\033[01;34m\]\w\[\033[00m\]\$ ' +else + PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ ' +fi +unset color_prompt force_color_prompt +# If this is an xterm set the title to user@host:dir +case "$TERM" in + xterm*|rxvt*) + PS1="\[\e]0;${debian_chroot:+($debian_chroot)}\u@\h: \w\a\]$PS1" + ;; +*) + ;; +esac +alias ls='ls -G' +alias ll='ls -alF' +alias la='ls -A' +alias l='ls -CF' +if [ -x /usr/bin/dircolors ]; then + test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)" + alias ls='echo "" && ls --color=auto && echo ""' + alias ls='ls --color=auto' + alias grep='grep --color=auto' + alias fgrep='fgrep --color=auto' + alias egrep='egrep --color=auto' +fi +alias alert='notify-send --urgency=low -i "$([ $? = 0 ] && echo terminal || echo error)" "$(history|tail -n1|sed -e '\''s/^\s*[0-9]\+\s*//;s/[;&|]\s*alert$//'\'')"' + +if [ -f ~/.bash_aliases ]; then + source ~/.bash_aliases +fi + +# de me + +alias vi="vim" +alias grep="grep --color=auto" +alias Grep="grep --color=auto" + +alias go-torch='PATH="${PATH}:$GOPATH/src/github.com/uber/go-torch/FlameGraph" go-torch' +alias golint="golint -min_confidence=.3" +alias git="git --no-pager" + +export PATH="${PATH}:$HOME/bin:$HOME/Library/Python/3.6/bin" +export PATH="${PATH}:/Applications/MongoDB.app/Contents/Resources/Vendor/mongodb/bin" +export VISUAL="vim" +PS1="\D{%a %d} \w $ " +PS1="\[\e[35m\]\w $\[\e[m\] " +PS1="\[\e[1m\e[7m\e[33m\]\w $\[\e[m\] " +PS1="\[\e[30;43m\]\w\[\e[m\]\[\e[30;43m\] \[\e[m\]\[\e[30;43m\]\\$\[\e[m\] " + +export PROMPT_COMMAND='echo -ne "\033]0;${USER}@${HOSTNAME}\007"' + +export GIT_EDITOR="vim" +export VISUAL="vim" +export EDITOR="$VISUAL" + +# FUNC + +function formalfile() { + local name="" + if [[ "$#" -lt 1 ]]; then + name="base" + else + name="$1" + fi + local f1=$(printf "%s.cpp" $name) + local f2=$(printf "%s.h" $name) + touch $f1 + touch $f2 + echo "/*" | tee -a $f1 $f2 + echo " * @file $f1" >> $f1 + echo " * @file $f2" >> $f2 + echo " * @author Bel LaPointe (lapoba16)" | tee -a $f1 $f2 + echo " * @date $(date +%Y/%m/%d)" | tee -a $f1 $f2 + echo " * @version 1.0" | tee -a $f1 $f2 + echo "$(printf " *\n */")" | tee -a $f1 $f2 + echo "$(printf "#include \"%s\"" $f2)" | tee -a $f1 + echo "$(printf "using namespace std;")" | tee -a $f1 + echo "$(printf "#include ")" | tee -a $f2 + echo "$(printf "#include ")" | tee -a $f2 + echo "$(printf "\n")" | tee -a $f1 $f2 + echo "$(printf "int main(int argc, char **argv) {")" | tee -a $f1 + echo "$(printf " return 0;")" | tee -a $f1 + echo "$(printf "}")" | tee -a $f1 + echo "$(printf "int main(int, char**);")" | tee -a $f2 +} +function bgrun() { + cmd="$*" + $cmd > /dev/null 2> /dev/null & +} +function mycd() { + cd ~/Go/src/gitlab-app.eng.qops.net/golang/qmp/ +} + +# GO +export GOPATH="$HOME/Go" +export GOROOT="/usr/local/Cellar/go/1.10/libexec" +export PATH="$PATH:$GOROOT/bin:$GOPATH/bin" +if [ ! -z "$GOROOT" ]; then + #echo "WARNING: GOROOT is set to "$GOROOT" but Go recommends nil" + GOROOT="" +fi + +# QSL/QPCL + +# MAVEN +export M2_HOME=/usr/local/Cellar/maven/3.3.9/libexec +export MVN_HOME=/usr/local/Cellar/maven/3.3.9/libexec +export MAVEN_OPTS="-Xms128m -Xmx1024m -ea -Duser.timezone=UTC" + +#export HISTSIZE=5000 +#export HISTFILESIZE=-1 + +# Docker +function docker() { + if [ "$1" == "run" ]; then + shift + /usr/local/bin/docker run --rm $* + else + /usr/local/bin/docker $* + fi + #clean_docker +} + +# CouchBase +for dc in "b1" "g1" "syd1" "fra1" "iad1" "ca1" "sjc1" "gov1"; do + alias couchbase${dc}="couchbaseweb $dc" +done + +function altgopath() { + #! /bin/bash + export GOPATH="$(cd "$(dirname "$BASH_SOURCE")"; pwd)" + export GOPATH=$(pwd) + mkdir -p $GOPATH/{bin,pkg,src} + export PATH=${PATH}:$GOPATH/bin +} + +export PATH="/usr/local/opt/gnu-sed/libexec/gnubin:$PATH" +GPATH=/usr/local/opt/coreutils/libexec/gnubin + +# QT5 +export PKG_CONFIG_PATH=${PKG_CONFIG_PATH}:/usr/local/opt/qt/lib/pkgconfig + +function ssh_gcp() { + ssh -i $HOME/.ssh/gcp_from_work gcp.blapointe.com -p 22 "$@" +} + +function ssh_tickle() { + /usr/bin/ssh -i $HOME/.ssh/id_rsa zach@192.168.1.123 "$@" +} + +function ssh_titus() { + /usr/bin/ssh titus@192.168.0.124 "$@" +} + +alias ssh='ssh -q -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oBatchMode=yes' +alias scp='scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null' + +function alldcs() { + local dcs=("a1" "b1" "g1" "gov1" "syd1" "fra1" "ca1" "sjc1" "iad1") + echo ${dcs[@]} | tr ' ' '\n' +} + +function allservices() { + local services=(isolation-service gobs marauders-map geni-mongo geni-mongo-arbiter-a fieldsets-db) + echo ${services[@]} | tr ' ' '\n' +} + +function getnodesindc() { + local dc="${1}" + local pattern="${2}" + shift + shift + if [ -z "$pattern" ] || [ -z "$dc" ]; then + echo "USAGE: this " >&2 + return 1 + fi + local out="" + if [ -e /tmp/getnodesindc.$dc ]; then + out="$(cat /tmp/getnodesindc.$dc)" + else + out="$(ssh "$dc" \ + curl http://consul.service.consul:8500/v1/catalog/nodes 2> /dev/null \ + | jq -r '.[].Node' \ + | sort -u \ + )" + printf "$out\n" > /tmp/getnodesindc.$dc + fi + printf "$out\n" | grep --color=none "${pattern:-".*"}" + return 0 +} + +function getfqdnsindc() { + local dc="${1}" + local service="${2}" + shift + shift + if [ -z "$service" ] || [ -z "$dc" ]; then + echo "USAGE: this " >&2 + return 1 + fi + if [ -z "${CLEAN:-""}" ] && [ "$#" == 0 ] && [ -e /tmp/getfqdnsindc.$dc.$service ]; then + cat /tmp/getfqdnsindc.$dc.$service + return 0 + fi + local out="$( + ssh \ + -q -oStrictHostKeyChecking=no -oUserKnownHostsFile=/dev/null -oBatchMode=yes \ + "$dc" \ + curl -sS http://consul.service.consul:8500/v1/health/service/$service"$@" \ + | jq -r '.[] | .Node.Node' \ + | sort -u)" + if [ "$#" == 0 ]; then + printf "$out\n" > /tmp/getfqdnsindc.$dc.$service + fi + printf "$out\n" + return 0 +} + + +function get_addr_dc_service() { + get_addrs_dc_service "$@" \ + | head -n 1 +} + +function watch_get_addrs_dc_service() { + local delay=1 + case "$1" in + b1*) ;; + g1*) ;; + *) delay=5 ;; + esac + watch -n $delay bash -c "true; echo $@; source $HOME/.bashrc; out=\$(get_addrs_dc_service $@); printf '%s\n' \"\$out\" | wc -l; printf '%s' \"\$out\"" +} + +function get_addrs_dc_service() { + for i in $(COMPACT=1 consul_healthy "$1" "$2" | grep -v === | sed 's/\./.preferred./'); do + case "$i" in + HEALTHY* ) ;; + UNHEALTHY* ) if [ "$#" -lt 3 ]; then return; fi ;; + * ) echo "$i" ;; + esac + done +} + +function watch_consul_healthy() { + if [ "$#" -lt 2 ]; then + echo "USAGE: this " + return 1 + fi + local dc="$1" + shift + while true; do + local new_out="$( + echo ===$dc===$@=== + echo "" + COMPACT=$SHORT$COMPACT consul_healthy $dc "$@" + echo "" + )" + clear + printf "%s\n" "$new_out" + sleep ${INTERVAL:-5} + done +} + +function set_secret() { + local key="$1" + read -s -p "Enter the secret '$key': " value >&2 + security delete-generic-password -a "${USER}" -s "$key" 2> /dev/null 1>&2 + security add-generic-password -a "${USER}" -s "$key" -w "$value" 1>&2 + echo "" >&2 +} + +function get_secret() { + local key="$1" + local value="$(security find-generic-password -a "${USER}" -s "$key" -w 2> /dev/null)" + if [ -z "$value" ]; then + set_secret "$1" + get_secret "$1" + return + fi + echo "$value" +} + +function mshell_() { + if [ $# -lt 3 ]; then + echo USAGE: mshell_ \$dc \$host \$port >&2 + return 1 + fi + local dc=$1 + local host=$2 + local port=$3 + shift + shift + shift + local container="$(ssh $host docker ps | grep unsupported | grep mongo | grep -v mom | head -n 1 | awk '{print $NF}')" + container=${container#*-} + container=${container:-.} + container="$(ssh $host docker ps | grep unsupp | grep $container | awk '{print $NF}' | head -n 1)" + if [ -z "$MONGO_SHELL_ROOT_PW" ]; then + MONGO_SHELL_ROOT_PW="$(get_secret mshell_root_pass)" + fi + local base_cmd=' \ + ssh $host \ + docker exec -i \ + $container \ + mongo \ + -u root \ + -p $MONGO_SHELL_ROOT_PW \ + --authenticationDatabase admin \ + --quiet \ + --port $port' + if [ $# -gt 0 ]; then + #echo "$@" | eval "$base_cmd" + echo "$@" | eval "$base_cmd" + else + base_cmd="${base_cmd//-i/-it}" + base_cmd="${base_cmd//ssh/ssh -t}" + eval "$base_cmd" + fi +} + +function clean_docker() { + for i in $(/usr/local/bin/docker images -qa -f "dangling=true") + do + for j in $(/usr/local/bin/docker container ls -a | grep "$i" | awk '{ print $1 }') + do + /usr/local/bin/docker container rm -f "$j" 2>&1 > /dev/null + done + /usr/local/bin/docker rmi -f "$i" 2>&1 > /dev/null + done + docker volume rm $(docker volume ls -qf dangling=true) + yes | docker volume prune + yes | docker network prune +} + +function clean_ssh () { + for i in $( \ + lsof -a -i4 -P -c '/^ssh$/' -s TCP:LISTEN \ + | awk \ + '{$1=""; $3=""; $4=""; $5=""; $6=""; $7=""; $8=""; $NF=""; print $0}' \ + | sed -E 's/(^[ ]*|[\t\s ]*$)//' \ + | sed 's/[ ][ ]*/_/g' \ + | grep -v PID \ + ); do + local pid="${i%%_*}" + printf '\033[?7l' + echo $i + printf '\033[?7h' + printf "Kill %d? (y/n) " "$pid" + while ((1)); do + read -n 1 opt + case "$opt" in + y) + kill "$pid" + echo " Killed $i" + break + ;; + n) + echo " Leaving $i" + break + ;; + q) + echo "" + return + ;; + esac + done + done +} + +#### IMPORTED +function consul_kv_get () { + local dc="${1:-"g1-iad"}" + local key="${2#/}" + ssh $dc \ + curl consul.service.consul:8500/v1/kv/$key 2> /dev/null \ + < /dev/null \ + | jq . \ + | jq -r '.[] | .Value' \ + | base64 --decode + echo "" +} + +function consul_kv_set() { + local dc="$1" + local key="${2#/}" + shift + shift + local content="$*" + if [ "$#" == 0 ]; then + content="$(cat)" + fi + if ! echo "${content}" | ssh $dc \ + curl -X PUT consul.service.consul:8500/v1/kv/$key \ + -d @- \ + 2> /dev/null \ + | grep -q true \ + ; then + return 1 + fi + consul_kv_get "$dc" "$key" +} + +function consul_role_dc () { + local role="$1" + local dc="$2" + shift + shift + ssh "$dc" curl "http://consul.service.consul:8500/v1/health/service/$service?$@" 2> /dev/null | jq -r '.[].Node.Node' | sort -u +} + +function couchbaseweb () { + port=$(getport) + local node=$(get_addr_dc_service $1 ${2:-gobs-db} | sed 's/:.*//' ) + if [ -z "$node" ]; then + echo no node found for $1 ${2:-gobs-db} + return 1 + fi + local dc="${node#*.}" + dc="${dc%%.*}" + cmd="(until curl --fail localhost:$port; do sleep 1; done; open http://localhost:$port) & ssh -NL $port:$node:8091 $dc" + echo "$cmd" + function clean() { + kill $(jobs -p) + } + trap clean EXIT SIGINT + eval "$cmd" +} + +function editGrepped () { + files=() + for i in $(grep -Ri $* . | awk '{ print $1 }' | grep -v "\.git" | grep -v Binary | sed 's/:.*//' | sort -u) + do + i="${i%%:*}" + files+=("$i") + done + vim "+/${1}" "${files[@]}" +} + +function editgrepped () { + editGrepped "$@" +} + +function gitgraph () { + num=5 + if [[ "$*" -gt 0 ]]; then + num=$1 + fi + git --no-pager log --graph --oneline -n $num +} + +function gitlog () { + let winwid=$(tput cols)-6 + if [[ $winwid -gt 120 ]]; then + winwid="120" + fi + format="" + format+="%Cgreen%h @ %an @ %ae%Creset" + format+="%n%C(Magenta) %ar (%ad)%Creset" + format+="%n%n%w($winwid,10,10)%s%n%w($winwid,10,10)%b%n" + num=2 + branch="" + if [[ "$#" -gt 0 ]]; then + num="$1" + fi + if [[ "$#" -gt 1 ]]; then + branch="$2" + fi + git --no-pager --no-pager log --date=short --pretty=format:"$format" -n $num $branch +} + +function gitmend () { + #gocheck + git --no-pager add -A :/ + if [ -n "$*" ]; then + git --no-pager commit --allow-empty --amend --date="$(date -R)" -m "$@" + else + git --no-pager commit --allow-empty --amend --date="$(date -R)" + fi +} + +function gitdiff() { + for excess in "" "--cached"; do + git --no-pager diff $excess "$@" $( + for f in $(git diff --name-only "$@" | grep -v -E '_test.go$|(^|\/)test_|go\.mod|go\.sum'); do + if [ -f $f ]; then + echo $f + fi + done + ) + done +} + +function gitmmit () { + gocheck + git --no-pager add -A :/ + git --no-pager commit -a -m "$*" +} + +function gitreset () { + git --no-pager reset --hard + git --no-pager clean -f -d +} + +function gitshrink () { + git --no-pager gc --aggressive --prune +} + +function gocheck () { + if [ "$(ls ./*.go 2> /dev/null)" != "" ]; then + if [ "$(go vet && golint)" != "" ]; then + echo "ERR go vet and golint" + fi + fi +} + +function goplaysh () { + cd $GOPATH/src +} + +function gotest () { + filename="gotestprofileout" + go test -covermode=count -coverprofile $filename "$@" + go tool cover -html=$filename && rm ./$filename +} + +function gotestcoverage () { + a=$(mktemp cover.XXXXXX) + b=$(mktemp cover.XXXXXX) + c=$(mktemp cover.XXXXXX) + go test -v -coverprofile=$a ./... 2>&1 > /dev/null && go tool cover -func=$a -o $c + lines="$(tail -1 $c)" + per="$(echo "$lines" | awk '{gsub("%","",$3); print $3}')" + per="$(echo "$per" | awk '{print $NF}')" + echo "${per}% tested" + rm $a $b $c +} + +function goveragetest () { + goverage -coverprofile myspecialsecretout.html ./... && go tool cover -html=myspecialsecretout.html && rm myspecialsecretout.html +} + +function goworksh () { + cd $GOPATH/src/gitlab-app.eng.qops.net +} + +function grepnovendor () { + if [ "$#" -lt 1 ]; then + echo "Need arguement" + return 1 + fi + grep --color=auto -E -Ri "$*" $(find . -mindepth 1 -maxdepth 1 | grep -v vendor | grep -v \./\.git) +} + +function purge_go_exec_from_git () { + java -jar /Users/breel/Downloads/bfg-1.13.0.jar ${@:- --delete-files $(basename $PWD)} && git --no-pager reflog expire --expire=now --all && git --no-pager gc --prune=now --aggressive + rm -rf ../$(basename $PWD).bfg-reportls +} + +function smoke_gobs () { + pushd $GOPATH/src/gitlab-app.eng.qops.net/data-pipeline/gobs/build + cmd="QTOKEN='$(get_qtoken b1)' python3 ./reqService.py --url gobs3-app.b1-prv.qops.net:8192/gobs/v1/types/fieldset-definitions/objects/" + echo "$cmd" + eval "$cmd" + popd +} + +function smoke_mm () { + pushd $GOPATH/src/gitlab-app.eng.qops.net/surveys/marauders-map/smoketests + cmd="HOSTNAME="$(hostname)" QTOKEN='$(get_qtoken b1)' python3 ./smoketests.py --hostname breellocal --local marauders-map3-app.b1-prv.qops.net --mongo_pwd p750kU39etYoqWw" + echo "$cmd" + eval "$cmd" + popd +} + +function touch_date () { + title="$*" + title="${title// /_}" + touch $(date "+%Y-%m-%d_$title").md +} + +function vendorupdate () { + govendor fetch ./... + govendor add +external +} + +alias rundeckgov1='open https://team-rundeck.gov1.qprod.net:4444 && ssh -L3000:team-rundeck:3000 -L4443:team-rundeck:4443 -L4444:team-rundeck:4444 -L4445:team-rundeck:4445 -L4446:team-rundeck:4446 -L4447:team-rundeck:4447 -L4448:team-rundeck:4448 -L4449:team-rundeck:4449 -L4450:team-rundeck:4450 -L4451:team-rundeck:4451 -L4452:team-rundeck:4452 -L4453:team-rundeck:4453 -L4454:team-rundeck:4454 -L4455:team-rundeck:4455 gov1' +alias rundeckgov1='ssh -NL 4444:team-rundeck1-app:4444 gov1 & until curl -k https://team-rundeck1-app.gov1.qprod.net:4444/; do sleep 1; done; open https://team-rundeck1-app.gov1.qprod.net:4444/; fg' +alias rundeckgov1is='open https://team-rundeck.gov1.qprod.net:4455 && ssh -L3000:team-rundeck:3000 -L4443:team-rundeck:4443 -L4444:team-rundeck:4444 -L4445:team-rundeck:4445 -L4446:team-rundeck:4446 -L4447:team-rundeck:4447 -L4448:team-rundeck:4448 -L4449:team-rundeck:4449 -L4450:team-rundeck:4450 -L4451:team-rundeck:4451 -L4452:team-rundeck:4452 -L4453:team-rundeck:4453 -L4454:team-rundeck:4454 -L4455:team-rundeck:4455 gov1' +alias uchiwagov1='open https://uchiwa-app.gov1.qprod.net:8443 && ssh -NL 8443:uchiwa-app:443 rampart.gov1.qprod.net' +alias vi='vim' +alias nexposegov1='open https://nexpose-console-app.gov1.qprod.net:49160 && ssh -NL 49160:nexpose-console-app:49160 gov1' +alias nomadgov1='open http://nomad-ui.gov1.qprod.net:3000/ && ssh -NL nomad-ui.gov1.qprod.net:3000:nomad-ui.service.consul:3000 gov1' +alias ffgov1='open https://gov-features.gov1.qprod.net:4443 && ssh -NL 4443:gov-features.gov1.qprod.net:443 gov1' +alias qameljobuigov1='open https://qamel-job-ui.gov1.qprod.net:4444 && ssh -NL 4444:qamel-job-ui.gov1.qprod.net:443 gov1' +qamelgov1() { + echo "LDAP password: '$(get_secret LDAP)'" + LAN_IP=$(ifconfig en0 | grep -o '[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*' | head -n 1) + LAN_PORT=9999 + set -x + ssh -NL $LAN_IP:$LAN_PORT:qamel-driver.service.gov1.consul:443 gov1 +} +alias grep='grep --color=auto -E' +alias grafanagov1='open https://grafana.gov1.qprod.net:53000/login && ssh -NL 53000:grafana1-app.gov1.qprod.net:443 gov1' +alias golint='golint -min_confidence=.3' +alias git='git --no-pager' +alias consulgov1='open http://consul1-app.gov1.qprod.net:8500/ && ssh -NL 8500:consul1-app:8500 gov1' +alias logviewgov1='open http://localhost:8989 && ssh -NL 8989:logging-local-bucket1:8989 gov1' +alias admintoolsgov1='open https://localhost:28091 && ssh -NL 28091:responses-tools1-app.gov1.qprod.net:443 responses-tools1-app.gov1.qprod.net' + +#complete -C /Users/breel/Go/src/github.com/minio/mc/mc mc +#complete -C /Users/breel/Go/bin/mc mc + +function qameltogov1tunnel() { + LAN_IP=$(ifconfig en0 | grep -o '[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*\.[0-9][0-9]*' | head -n 1) + LAN_PORT=9999 + local branch="$1" + shift + qamel "$branch" -H $LAN_IP -P $LAN_PORT "$@" +} + +function qamel() { + function clean() { + while popd 2> /dev/null; do + true + done + } + local QAMEL_SCM_PAT=${QAMEL_SCM_PAT} + if [ -z "$QAMEL_SCM_PAT" ]; then + QAMEL_SCM_PAT=$(get_secret QAMEL_SCM_PAT) #dvQFsMaLTVqmi8FRMGTb + fi + local tag="$1" + if [ -z "$(git tag | grep "$tag")" ] && [ -z "$(git branch | grep "$tag")" ]; then + echo "Tag/branch $tag not found" >&2 + return 1 + fi + shift + until [ -d ./.git ]; do + if ! pushd .. 2> /dev/null; then + clean + return 1 + fi + done + local path="$(cat .git/config \ + | grep url \ + | sed 's/^.*net[^\/]*\///' \ + | sed 's/.git$//' \ + | sed 's/\//%2F/g' \ + )" + local id="$(set -x; curl \ + -H Content-Type:application/json \ + -H PRIVATE-TOKEN:$QAMEL_SCM_PAT \ + https://gitlab-app.eng.qops.net/api/v4/projects/${path#/} \ + 2> /dev/null \ + | jq -r .id \ + )" + local id="${path//%2F/\/}" + if [ -z "$id" ]; then + echo "Cannot find project id for $path" >&2 + clean + return 1 + fi + local args=("$@") + local use_args=() + local has_ts=0 + local has_assignee=0 + local has_tag=0 + local is_submit_job=0 + for ((i=0; i<${#args[@]}; i++)); do + local arg="${args[i]}" + if [ -e "$arg" ]; then + arg="gitlab:yaml:${id}:${tag}:${arg#./}" + fi + if [ "${arg%:*}" == "PC_TICKET_ASSIGNEE" ]; then + has_assignee=1 + fi + if [ "${arg%:*}" == "TS" ]; then + has_ts=1 + fi + if [ "${arg%:*}" == "TAG" ]; then + has_tag=1 + fi + if [ "$arg" == submit-job ]; then + is_submit_job=1 + fi + use_args+=("$arg") + done + if [ "$has_ts" == 0 ] && [ "$is_submit_job" == 1 ]; then + use_args+=("--restricted-variable") + use_args+=("TS:$(date "+%Y-%m-%d_%H:%M_%Z")") + fi + if [ "$has_assignee" == 0 ] && [ "$is_submit_job" == 1 ]; then + use_args+=("--restricted-variable") + use_args+=("PC_TICKET_ASSIGNEE:breel") + fi + if [ "$has_tag" == 0 ] && [ "$is_submit_job" == 1 ]; then + use_args+=("--restricted-variable") + use_args+=("TAG:$tag") + fi + use_args+=("--restricted-variable" "NEED_FOR_DILIGENCE:low") + use_args+=("--restricted-variable" "ASSIGNEE:breel") + use_args+=("--restricted-variable" "REVIEWER:breel") + use_args+=("--restricted-variable" "CHANGELOG:changes") + use_args+=("--restricted-variable" "CUSTOMER_IMPACT:low") + use_args+=("--restricted-variable" "OUTAGE_RISK:low") + echo QAMEL_SCM_PAT=$QAMEL_SCM_PAT qamel-driver \ + "${use_args[@]}" \ + --scm-pat-from-environment >&2 + QAMEL_SCM_PAT=$QAMEL_SCM_PAT qamel-driver \ + "${use_args[@]}" \ + --scm-pat-from-environment + clean +} + +function yaml_to_json() { + python3 -c 'import sys, yaml, json; json.dump(yaml.load(sys.stdin), sys.stdout, indent=3)' +} + +function get_geni_logs() { + if [ "$#" -lt 1 ]; then + echo "USAGE this.sh \$dc [ \$date ]" >&2 + return 1 + fi + local dc="$1" + local base="https://log-view.eng.qops.net/logs/$dc/storage-platform-1/geni/access" + #echo "$base" >&2 + local date="${2:-$( \ + curl -L $base \ + 2> /dev/null \ + | grep -o 'href..\.\/[^\/]*\/' \ + | grep -o '\.\/[^\/]*\/' \ + | grep -Eo '[0-9]{4}-[0-9]{2}-[0-9]{2}' \ + | sort \ + | tail -n 1 \ + )}" + local logs=($( \ + curl -L $base/$date/ \ + 2> /dev/null \ + | grep -o 'href.*.json' \ + | grep -o '\.\/[^"]*' \ + )) + + #echo $dc $date ${logs[@]} >&2 + for log in ${logs[@]}; do + local link="$base/$date/$log" + curl $link 2> /dev/null \ + | sed 's/^[^{]*//' \ + | jq -r .message \ + | sed 's/^[^{]*//' + #| sed 's/\\"/"/g' + + read -p "More? [Y/n]" yesno >&2 + case "$yesno" in + n* ) break;; + N* ) break;; + Y* ) ;; + y* ) ;; + * ) return 1;; + esac + done +} + +export RE_CODE_LOC=$HOME/repos/responseengine + +function tail_logs() { + local dc="$1" + local service="$2" + if [ -z "$dc" ] || [ -z "$service" ]; then + echo "USAGE this.sh \$dc \$service" >&2 + return 1 + fi + trap kill_jobs SIGINT + for node in $(consul_healthy $dc $service | grep \\.net); do + tail_log "$node" & + done + read stopnow + kill_jobs +} + +function tail_log() { + local node="$1" + local path="$2" + if [ -z "$path" ] || [ -z "$node" ]; then + echo "USAGE this.sh \$fqdn \$path" >&2 + return 1 + fi + echo NOT IMPL "ssh -t -t $node bash -c 'true; tail -f '"$path" < /dev/null" >&2 + return 1 +} + +function tail_nomad_logs() { + local dc="$1" + local service="$2" + if [ -z "$dc" ] || [ -z "$service" ]; then + echo "USAGE this.sh \$dc \$service" >&2 + return 1 + fi + trap kill_jobs SIGINT + for node in $(consul_healthy $dc $service | grep \\.net); do + tail_nomad_log "$service" "$node" & + done + read stopnow + kill_jobs +} + +function kill_jobs() { + for job in $(jobs | awk '{print $1}' | grep -o '[0-9][0-9]*'); do + eval "kill %${job}" + done + jobs +} + +function tail_nomad_log() { + local service="$1" + local node="$2" + if [ -z "$service" ] || [ -z "$node" ]; then + echo "USAGE this.sh \$service \$fqdn" >&2 + return 1 + fi + local c="$(nomad_container "$service" "$node")" + local uuid="${c#${service}-}" + if [ -z "$uuid" ]; then + return 1 + fi + ssh -t -t $node bash -c 'true; f=$(ls /var/lib/nomad/alloc/'"$uuid"'/alloc/logs/*access*); tail -f $f' < /dev/null +} + +function nomad_container() { + local service="$1" + local node="$2" + if [ -z "$service" ] || [ -z "$node" ]; then + echo "USAGE this.sh \$service \$fqdn" >&2 + return 1 + fi + ssh -t -t "$node" docker ps < /dev/null \ + | awk '{print $NF}' \ + | grep "$service.[0-9A-Za-z]{8}.[0-9A-Za-z]{4}.[0-9A-Za-z]{4}.[0-9A-Za-z]{4}.[0-9A-Za-z]{12}" \ + | head -n 1 +} + +function relgodoc() { + local path="$1" + shift + path="$(realpath "$path")" + path="${path#$GOPATH/src/}" + go doc "${path%/}" "$@" +} + +alias g3ssh='ssh 54.186.183.151' +fluentview() { + local extra="" + if echo " $* " | grep -E -q '[\s\b ]-+dc(=|[\s\b ]+)gov1'; then + extra="-awsu=AKIAU4KA6LB32HZQW2BK -awsp=G2TEZqJB0rBOZYkBoAVw9djEWk3R1vatu2fZTg9S" + fi + $(which fluentview) \ + -storeaddr /var/log/fluentview/local.db \ + -awsu AKIA4TVJAYO66A4K34B2 \ + -awsp jWEf/oSKM6tt7a3ofkl6OFpJY/FDW1tzrBTVp/84 \ + -n 10 \ + -parse \ + "$@" \ + $extra +} +alias curl='curl -sS' + +function vaultb1() { + local path="$1" + shift + if [ -z "$path" ]; then + echo "USAGE: this.sh secret/path/to/secret/directory [additional jwtcurl params]" >&2 + return 1 + fi + jwtcurl \ + -jwt=false \ + -headers X-Vault-Token=${TOKEN:-6d3891df-8a1d-4f04-b2a0-de75dee0b2cd} \ + -host https://vault.${DC:-b1-prv}.qops.net:8200 \ + -path "/v1/secret/${path#*secret/}" \ + "$@" +} + +function linux_go_build() { + GOOS=linux CGO_ENABLED=0 \ + go build \ + -o $GOPATH/bin/linux/exec-${PWD##*/} \ + -a \ + -installsuffix cgo \ + -ldflags "-s -w -X main.GitCommit=$( + ( + git rev-list -1 HEAD + if git diff | grep . > /dev/null; then + echo "-dirty" + fi + ) 2> /dev/null | tr -d '\n' + )" +} +function remote_jwtcurl() { + bash $HOME/bin/work/remote_jwtcurl "$@" +} +function rsid_to_fsid() { + local dc="$1" + local rsid="$2" + remote_jwtcurl "$dc" -path /fieldsetdefinitions/admin/index/responseset/$rsid/fieldsets \ + | jq -r '.values[]' +} +function fsid_to_brandid() { + local dc="$1" + local fsid="$2" + remote_jwtcurl "$dc" -path /fieldsetdefinitions/admin/index/fieldsets/$fsid/brand \ + | jq -r '.values[]' +} +function fsid_to_smids() { + local dc="$1" + local fsid="$2" + remote_jwtcurl "$dc" -path /fieldsetdefinitions/v1/index/fieldsets/$fsid/sourcemaps \ + | jq -r '.values[]' +} +function mstate_() { + local dc="$1" + if [ -z "$dc" ]; then + echo "[ERROR] Usage: this.sh dc [shard]" >&2 + return 1 + fi + local shard="${2:-a}" + shard="geni-mongo-${shard#geni-mongo-}" + local np=$(get_addr_dc_service "$dc" "$shard") + if [ -z "$np" ]; then + echo "[ERROR] cannot find node for $dc $shard" >&2 + return 1 + fi + local p=${np##*:} + local n=${np%%.*} + mshell_ \ + $dc $n $p \ + 'rs.status().members.forEach(function(e) { if ("optime" in e) print(e["name"]+":"+e["stateStr"]+":"+e["optime"]["ts"]); });' +} + +function truncate() { + cut -c 1-${1:-300} +} +function fsid_to_rsids() { + if [ $# -lt 2 ]; then + echo "USAGE: this dc fsid" >&2 + return 1 + fi + remote_jwtcurl $1 -path fieldsetdefinitions/admin/index/fieldsets/$2/responseset \ + 2> /dev/null \ + | jq -r '.values[]' +} +function consul_services() { + local dc="$1" + local prefix="" + if [ -n "$dc" ]; then + prefix="ssh $dc" + fi + $prefix curl consul.service.consul:8500/v1/catalog/services 2> /dev/null | jq -r 'keys[]' +} + +function tunnelcurl() { + local dc="$1" + local host="$2" + local port="$3" + shift + shift + shift + docker rm -f tunnelbox > /dev/null 2>&1 + docker run \ + -v $HOME/.ssh:/root/.ssh \ + -d \ + --name tunnelbox \ + tunnelbox:latest \ + $port:$host:$port $dc \ + + #> /dev/null + until docker exec -i tunnelbox \ + curl $host:$port"$@" 2> /dev/null; do + sleep 1 + done + docker rm -f tunnelbox > /dev/null 2>&1 +} + +export PATH="$PATH:$HOME/bin/jdk8u222-b10/Contents/Home/bin" +export PATH="$PATH:/usr/local/opt/mysql-client/bin" +export PATH="$HOME/bin/work:$PATH" +#export JAVA_HOME="$HOME/bin/jdk8u222-b10/Contents/Home" +jenv add "$HOME/bin/jdk8u222-b10/Contents/Home" > /dev/null 2>&1 + + +function pprof() { + local host=$1 + local path=$2 + wget -O /tmp/$path $host:6060/debug/pprof/$path + go tool pprof -http=:$(getport) /tmp/$path +} + +function old_rabbit_queue_ls() { + if [ "$#" -lt 1 ]; then + echo "USAGE: this " >&2 + return 1 + fi + local dc="$1" + remote_jwtcurl \ + $dc \ + -host $(get_addr_dc_service $dc response-rabbitmq) \ + -path api/queues \ + -auth responses,Re14sponses +} + +export GOFLAGS=-mod=vendor +export GO111MODULE=off +export GOPRIVATE=gitlab-app.eng.qops.net/* +export GOPROXY=https://goproxy.eng.qops.net:1720 +export GONOPROXY=none +export GONOSUMDB=gitlab-app.eng.qops.net/* +export QTOKEN=dnKgzTPNZyEd2Kfop +export QTOKEN='wrAiRgxzh27Id3iy58@ra5EehNyXDEsfANtFf' + +get_qtoken() { + bash $HOME/bin/work/get_qtoken "$@" +} +if dig +short consul.service.consul | grep -q .; then + get_qtoken b1 &> /dev/null & +fi & +disown + +jwtcurl() { + $(which jwtcurl) -issuer responseengine,blixt -secret "$(get_qtoken b1)" -t 100h "$@" +} + +set -o pipefail + +export CLASSPATH="$PATH;$CLASSPATH;." +export PATH="/usr/local/opt/ruby/bin:$PATH" +export PATH="$HOME/.jenv/bin:$PATH" +export JENV_ROOT=/usr/local/opt/jenv +( +if which jenv > /dev/null; then eval "$(jenv init -)"; fi +jenv enable-plugin maven > /dev/null 2>&1 +#export JAVA_HOME=/Library/Java/JavaVirtualMachines/adoptopenjdk-11.jdk/Contents/Home +jenv add "/Library/Java/JavaVirtualMachines/adoptopenjdk-11.jdk/Contents/Home" > /dev/null 2>&1 +jenv add "/Library/Java/JavaVirtualMachines/adoptopenjdk-8.jdk/Contents/Home" > /dev/null 2>&1 +) &> /dev/null & +disown +export PATH="/Library/Java/JavaVirtualMachines/adoptopenjdk-11.jdk/Contents/Home/bin:$PATH" +export PATH="/usr/local/opt/mongodb-community@4.2/bin:$PATH" +export PATH="$PATH:$(echo /usr/local/lib/ruby/gems/*/bin)" +export PATH="/usr/local/Cellar/python@3.8/3.8.6/Frameworks/Python.framework/Versions/3.8/bin:$PATH" +export PATH="/usr/local/Cellar/node/15.0.1/libexec/bin:$PATH" +export PATH="/usr/local/Cellar/node/15.0.1/bin:$PATH" +export PATH="/usr/local/go/bin:$PATH" +export PATH="$PATH:/Applications/Couchbase Server.app/Contents/Resources/couchbase-core/bin" + +function python3() { + $(which python3) -B "$@" +} + +function git_big_files() { + git rev-list --all --objects \ + | sed -n $( + git rev-list --objects --all \ + | cut -f1 -d' ' \ + | git cat-file --batch-check \ + | grep blob \ + | sort -n -k 3 \ + | tail -n40 \ + | while read hash type size; do + echo -n "-e s/$hash/$size/p "; + done + ) \ + | sort -n -k1 +} + +function git_purge_file() { + export FILTER_BRANCH_SQUELCH_WARNING=1 + git filter-branch -f --index-filter \ + 'git rm -r --force --cached --ignore-unmatch "'"$@"'"' \ + -- --all \ + && rm -Rf .git/refs/original && \ + git reflog expire --expire=now --all && \ + git gc --aggressive && \ + git prune +} + +function concat_v() { + if [ "$#" -lt 1 ]; then + echo "USAGE: concat_v output.path input [input2] [input3]" >&2 + return 1 + fi + local out="$1" + shift + convert "$@" -append "$out" +} + +function sum_in_aws_s3_console() { + echo 'cnt = 0; sum = 0; Array.from(document.getElementsByTagName("td")).forEach(function(e) { if (e.className == "ng-binding ng-scope" && e.getAttribute("ng-if") == "obj.Size !== '"''"'") { t = e.innerHTML; n = t.split(" ")[0]; b = t.split(" ")[1]; if (b == "KB") n *= 1000; else n *= 1000*1000; console.log(n); sum += n; cnt += 1; } }); console.log("cnt =", cnt, ", sum =", sum/1000/1000/1000, "gb")' +} + +function md_to_nice() { + if [ "$#" -lt 2 ]; then + echo "USAGE: $0 extension 1.md 2.md..." >&2 + return 1 + fi + local ext="$1" + shift + local files=("$@") + local out="$(basename "${files%.md}").$ext" + pandoc -s -o "$out" "${files[@]}" && open "$out" +} +function b1minio() { + ssh gobs2-db.b1-prv.qops.net \ + docker \ + run \ + --rm \ + -d \ + -p 9000:9000 \ + -e MINIO_ACCESS_KEY=AKIAIFSKOGA722LP42SA \ + -e MINIO_SECRET_KEY=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 \ + --name minio \ + -v /var/lib/minio:/data \ + registry-app.b1-prv.qops.net:5001/devops/minio:latest \ + server /data +} + +function open_files() { + lsof | awk '{print $1}' | uniq -c | sort -rn | head +} + +docker_rmi_prompt() { + local yn="" + for img in $(docker images | awk '{print $1":"$2":"$3}'); do + if [[ "${img%:*}" == *":" ]]; then + yn="" + else + read -p "remove ${img%:*}? Y/n" yn + fi + if [ "$yn" != "n" ]; then + docker rmi -f "${img##*:}" + fi + done +} + +alias less='less -R' + +miniprompt() { + export PS1='\[\e[30;43m\]\[\e[m\]\[\e[30;43m\]\[\e[m\]\[\e[30;43m\]\$ \[\e[m\]' +} + +lssrc() { + ls "$@" | grep -v -E 'test_|main|init|_test' +} + +alias gomoduleson='export GOFLAGS=""; export GO111MODULE=""; export export CGO_CFLAGS_ALLOW="-Xpreprocessor"' +alias offgomodules='export GOFLAGS="-mod=vendor"; export GO111MODULE="off"' +gomoduleson + +alias pip=pip3 +rawurlencode() { + local string="${1}" + local strlen=${#string} + local encoded="" + local pos c o + + for (( pos=0 ; pos /dev/null; then + true || work-pttodo +fi + +source /Users/breel/Sync/Core/Todo/bin/functions.sh + +ezmdcd() { + cd /Users/breel/Go/src/local/containers/work-notes/notea/spike/review/reinvent/ezmded/server +} + +gitpullbs() { + for b in "$@"; do + until git checkout $b && git pull origin $b; do + sleep 1 + done + done +} + +shopt -s direxpand + +lines_between_patterns() { + local from="$1" + local to="$2" + cmd="awk '/${from//"/"/\\/}/,/${to//"/"/\\/}/'" + echo "> $cmd" >&2 + eval "$cmd" +} + +alias jq='gojq' +export NEXPOSE_BILLING_TEAM='datastore' diff --git a/.gitconfig b/.gitconfig new file mode 100644 index 0000000..32ba151 --- /dev/null +++ b/.gitconfig @@ -0,0 +1,3 @@ +[user] + email = bel@bel.bel + name = bel diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..68a0517 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +* +!*.* +!*/ + +**/*.sw* +/Library +/Go/bin +/Go/pkg +/.viminfo diff --git a/.vimrc b/.vimrc new file mode 100644 index 0000000..781e071 --- /dev/null +++ b/.vimrc @@ -0,0 +1,64 @@ +inoremap { {}ko +inoremap ( ()ko +"inoremap { {}ha i + +set ruler +set hlsearch +syntax enable + +set shiftwidth=3 +set tabstop=3 +autocmd Filetype * setlocal ts=3 sts=3 sw=3 expandtab +set expandtab +set number +set relativenumber +set autoindent + +" Comment formatting for C/C++/CSS +set formatoptions+=r + +"set nowrap + +" cleaner wrap +set breakindent + +" case insensitive default (goes sensitive if there's a capital letter) +set ignorecase +set smartcase + +" nice word wrapping +set wrap +set linebreak + +"set mouse=a +"if has("mouse_sgr") +" set ttymouse=sgr +"else +" set ttymouse=xterm2 +"end + +" goimports on save +autocmd BufWritePre * %s/\([^ ]\) $/\1/e +autocmd BufWritePre * %s/\s\s\s\+$//e +autocmd BufWritePost *.go !timeout 2 goimports -w % +autocmd BufRead,BufNewFile *.go set autoread +"set rtp+=timeout 2 $GOPATH/src/github.com/golang/lint/misc/vim +"autocmd BufWritePost,FileWritePost *.go execute 'Lint' | cwindow + +autocmd Filetype java setlocal ts=4 sts=4 sw=4 expandtab +autocmd Filetype php setlocal ts=4 sts=4 sw=4 noexpandtab +autocmd Filetype yaml setlocal ts=2 sts=2 sw=2 expandtab +autocmd Filetype eyaml setlocal ts=2 sts=2 sw=2 expandtab +autocmd Filetype yml setlocal ts=2 sts=2 sw=2 expandtab +autocmd Filetype rb setlocal ts=2 sts=2 sw=2 expandtab +autocmd Filetype Jenkinsfile setlocal ts=3 sts=3 sw=3 noexpandtab +autocmd Filetype js setlocal ts=4 sts=4 sw=4 noexpandtab +autocmd Filetype javascript setlocal ts=4 sts=4 sw=4 expandtab +autocmd Filetype typescript setlocal ts=2 sts=2 sw=2 expandtab +autocmd Filetype ts setlocal ts=2 sts=2 sw=2 expandtab + +set binary +set ffs=unix +autocmd FileType crontab setlocal nowritebackup + +"colors pyte diff --git a/env.sh b/env.sh new file mode 100644 index 0000000..17dfb18 --- /dev/null +++ b/env.sh @@ -0,0 +1,10 @@ +#! /bin/bash + +cd "$(dirname "$(realpath "$BASH_SOURCE")")" +mkdir -p ./Go/{src,bin,pkg} +export GOPATH="$PWD/Go" + +if [ "$PWD" != "$HOME" ]; then + cp $HOME/.{bash,vim}rc ./ + export HOME="$PWD" +fi diff --git a/log.yaml b/log.yaml new file mode 100644 index 0000000..c80a8e8 --- /dev/null +++ b/log.yaml @@ -0,0 +1,44 @@ +recv: + questions: + - what do you like about go? goroutines; if you can vertical scale then very readable + - what don't you like about go? channels; create confusion and open door to a mess + - when to use a buffered channel? when non-buffered? buffered only as an option and is limited by how fast consuming is anyway + - how do you like to sync if you dont like channels? channels are fine, mutexes and semaphores via channels are fine too + - how to `find` in mongo? .... find() + - how to `average` in mongo? without aggregate pipeline? pre-aggregate via a second collection/column/document + - how to shard `{user:1, ts:123, value:.8}`? by user:1,ts:1;; never hash! + - how to build an index? ... createIndex() + - would this use index intersection? `{user:5, ts: {gt:now-7days}}`? i said yes, he said no + - which index would it use? probably userid:x, since it won't head to see how much that is + - how to determine which index? .explain() +send: + questions: + - q: what's on your wishlist roadmap this year? for the next 5 years? + a: finish deprecation work !!! + + - q: where are SLAs defined? How are issues escalated? + a: we dont notice stuff until a customer has a big bug report;; we have grafana and prometheus !!! + + - q: do you feel comfortable discussing compensation with your lead? with one another? + a: yeah, but it has its cons + + - q: you're using mgo? + a: no go-common is too painful + + not asked: + - q: do you find yourself blocked by other teams? working in other teams' spaces? + - q: there's a github repo on running tidepool locally. Any opinions on it? Do you use it? + - q: how do you manage configs? secrets? + + - q: who's someone you look up to? what makes them awesome? + - q: what does planning vacation time look like? + - q: what is performance review process like? + + cut questions: + - q: what does a release look like? when is something -release ready-? + - q: what's code review like? timeline? protocol? + - q: what operating system do you work in? was it a choice? what software is standard? what do you wish for? (IDEs, VPNs, ...) + - q: what's your biggest worry? + - q: do developers have SSH/root access/db admin access? why? + - q: what's the DR plan? + - q: what automations do you have? CICD? Metrics and alerts? Deployment?