Zenn
🐚

Shell oneliners (updated from time to time)

2021/10/21に公開
Bash
shell
#
jq
#
sh
tech

Tail particular GitLab log with highlight

sudo gitlab-ctl tail
  | sed -nre'/^==> .*(sidekiq\/current|gitlab-rails\/application_json).* <==/,/^$/p' 
  | grep -E 'pipeline|$'

Below two combined

gitlab-ctl tail 
  | sed -nre'/^==> .*(prometheus).* <==/,/^$/d;/^==> .* <==/,/^$/p'
  | jq -Rr '. as $line | fromjson? // $line'

exclude a few component(s) from gitlab-ctl tail output

gitlab-ctl tail 
  | sed -nre'/^==> .*(prometheus).* <==/,/^$/d;/^==> .* <==/,/^$/p'

let jq process json lines and pass-through others

jq -Rr '. as $line | fromjson? // $line'

Get the latest tag of an image on Docker Hub

curl -sH "Authorization: Bearer
  $(curl -s "https://auth.docker.io/token?service=registry.docker.io&scope=repository:gitlab/gitlab-runner:pull"
    | jq -r .token)"
  "https://registry-1.docker.io/v2/gitlab/gitlab-runner/tags/list"
  | jq -r '.tags[]'
  | sort -V
  | tail -n 1

Pickup member add/remove events from GitLab group audit events

output in json

curl -sH "PRIVATE-TOKEN: $TOKEN" 'https://gitlab.com/api/v4/groups/$GROUP_ID/audit_events?per_page=100&page=1' \
  | jq '.[]|select((.details.add//.details.remove)=="user_access")|{(if .details.add then "add" else "remove" end):.details.target_details,"as":.details.as,"by":.details.author_name,"at":.created_at}' \
  | jq -s .

output in csv

curl -sH "PRIVATE-TOKEN: $TOKEN" 'https://gitlab.com/api/v4/groups/$GROUP_ID/audit_events?per_page=100&page=1' \
  | jq '.[]|select((.details.add//.details.remove)=="user_access")|[(if .details.add then "add" else "remove" end),.details.target_details,.details.as,.details.author_name,.created_at]' \
  | jq -r '.|@csv'

Watch unix socket communication (GitLab example)

strace -f -t -s 2014 -p $(ps -C gitlab-workhors --no-headers -o '%p') -e trace=read,write

Delete orphaned local Git branch

git fetch --prune &&
git branch -d $(git branch -v | sed -rne'/\[gone\]/s/^\*?\s+(\S+)\s.*$/\1/p')

Pagenated REST api in one go (GitLab API projects storage stats example)

TOKEN=Your-Personal-Token URL=https://your.gitlab.server n=1
while :
  do \
    x=$(curl -sH "PRIVATE-TOKEN: ${TOKEN}" "${URL}/api/v4/projects?statistics=true&per_page=100&page=$n")
    if [[ "$x" == "[]" ]]; then \
      unset TOKEN
      break
    fi
    echo $x | jq -c '.[] | {path: .path_with_namespace, stats: .statistics}'
    n=$((n+1))
  done

streamed version

TOKEN=Your-Personal-Token URL=https://your.gitlab.server n=1; while :; do x=$(curl -sH "PRIVATE-TOKEN: ${TOKEN}" "${URL}/api/v4/projects?statistics=true&per_page=100&page=$n"); if [[ "$x" == "[]" ]]; then unset TOKEN; break; fi; echo $x | jq -c '.[] | {path: .path_with_namespace, stats: .statistics}'; n=$((n+1)); done

to output in csv

Replace the line starting echo $x | with below

echo $x | jq -cr '.[] | [.path_with_namespace, .statistics.commit_count, .statistics.storage_size, .statistics.repository_size, .statistics.wiki_size, .statistics.lfs_objects_size, .statistics.job_artifacts_size, .statistics.snippets_size, .statistics.packages_size] | @csv'

Download Redmine project's issue PDFs in one go

curl -sH 'X-Redmine-API-Key:<Redmine API Key goes here>' https://redmine.example.com/projects/project_path/issues.csv?encoding=UTF-8 \
  | sed -rne's/^([0-9]*),.+$/\1/p' \
  | xargs -I {} -n 1 curl -sOJH 'X-Redmine-API-Key:<Redmine API Key goes here>' https://redmine.example.com/issues/{}.pdf

Count text file lines and chars in a git repository

find -type d -name .git -prune \
  -o -type f -exec sh -c "file -b {} | grep -Pq 'JSON|text'" \;
  -exec cat {} \; | wc

Search multiple archived logs and append file names each line of the output

ls *.log.1.gz | \
  xargs -n 1 -I {} sh -c '
    zcat -f {} | \
    sed -re"s/^/{}: /" \
  ' | \
  grep -iP '<search pattern>' | \
  less -I

View json log with jq formatted and with color

zcat whatever-json.log.1.gz | \
  grep -iP '<search pattern>' | \
  jq -C . | \
  less -RI

Discussion