diff --git a/.circleci/config.yml b/.circleci/config.yml index fc2492b..4ec21d4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,13 +5,13 @@ # # vim:ts=2:sts=2:sw=2:et # -# https://github.com/harisekhon/sql-scripts +# https://github.com/HariSekhon/SQL-scripts # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # https://circleci.com/docs/2.0/configuration-reference diff --git a/.editorconfig b/.editorconfig index 8327005..c8c23aa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -3,14 +3,14 @@ # Author: Hari Sekhon # Date: 2015-10-31 19:04:34 +0000 (Sat, 31 Oct 2015) # -# https://github.com/harisekhon/sql-scripts +# https://github.com/HariSekhon/SQL-scripts # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # http://EditorConfig.org @@ -40,7 +40,14 @@ end_of_line = lf trim_trailing_whitespace = true insert_final_newline = true -[*.md] +[{*.md,*.hcl,*.tf,*.tfvars}] +indent_size = 2 +indent_style = space +end_of_line = lf +trim_trailing_whitespace = true +insert_final_newline = true + +[*.yml,*.yaml] indent_size = 2 indent_style = space end_of_line = lf diff --git a/.envrc b/.envrc new file mode 100644 index 0000000..bd53543 --- /dev/null +++ b/.envrc @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/SQL-scripts +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# D i r E n v +# ============================================================================ # + +# https://direnv.net/man/direnv-stdlib.1.html + +# See Also: +# +# .envrc-aws +# .envrc-gcp +# .envrc-kubernetes + +# direnv stdlib - loads .envrc from parent dir up to / +# +# useful to accumulate parent and child directory .envrc settings eg. adding Kubernetes namespace, ArgoCD app etc. +# +# bypasses security authorization though - use with care +#source_up +# +# source_up must be loaded before set -u otherwise gets this error: +# +# direnv: loading .envrc +# /bin/bash: line 226: $1: unbound variable +# +# source_up causes this error is up .envrc is found in parent directories: +# +# direnv: No ancestor .envrc found + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +src="$(readlink -f "${BASH_SOURCE[0]}")" +srcdir="$(cd "$(dirname "$src")" && pwd)" + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +# Automatically install Pre-Commit Git hooks if not already present + +if ! type -P pre-commit &>/dev/null; then + if uname -s | grep -q Darwin && + type -P brew &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Homebrew..." + echo + brew install pre-commit + echo + elif type -P pip &>/dev/null; then + echo + echo "Pre-commit is not installed - installing now using Pip..." + echo + pip install pre-commit + fi +fi + +if [ -f .pre-commit-config.yaml ] && + type -P pre-commit &>/dev/null && + git rev-parse --is-inside-work-tree &>/dev/null; then + if ! [ -f "$(git rev-parse --show-toplevel)/.git/hooks/pre-commit" ]; then + echo + echo "Pre-commit hook is not installed in local Git repo checkout - installing now..." + echo + pre-commit install + fi +fi + +# ============================================================================ # +# D o c k e r C o m p o s e +# ============================================================================ # + +export COMPOSE_PROJECT_NAME="SQL-scripts" + +# ============================================================================ # +# G i t H u b +# ============================================================================ # + +#export GITHUB_ORGANIZATION=HariSekhon + +# ============================================================================ # +# A n s i b l e +# ============================================================================ # + +# use the local repo's ansible.cfg rather than: +# +# $PWD/ansible.cfg +# ~/.ansible.cfg +# /etc/ansible/ansible.cfg +# +# set this in project repos to ensure user environment ANSIBLE_CONFIG doesn't get used +#export ANSIBLE_CONFIG="/path/to/ansible.cfg" + +# ============================================================================ # +# C l o u d f l a r e +# ============================================================================ # + +#export CLOUDFLARE_EMAIL=hari@... +#export CLOUDFLARE_API_KEY=... # generate here: https://dash.cloudflare.com/profile/api-tokens +#export CLOUDFLARE_TOKEN=... # used by cloudflare_api.sh but not by terraform module + +# export the variables for terraform +#export TF_VAR_cloudflare_email="$CLOUDFLARE_EMAIL" +#export TF_VAR_cloudflare_api_key="$CLOUDFLARE_API_KEY" # must be a key, not a token using the link above + +# ============================================================================ # +# Load External Envrc Files If Present +# ============================================================================ # + +# XXX: safer to bring all these external .envrc inline if you're worried about changes +# to it bypassing 'direnv allow' authorization +load_if_exists(){ + # first arg is a path to a .envrc + # all other args are passed to the sourcing of .envrc - used by .envrc-kubernetes + # to pass the context name 'docker-desktop' to switch to + local envrc="$1" + shift + if ! [[ "$envrc" =~ ^/ ]]; then + envrc="$srcdir/$envrc" + fi + if [ -f "$envrc" ]; then + # prevent looping on symlinks to this .envrc if given + if [ "$(readlink "$envrc")" = "$src" ]; then + return + fi + echo + echo "Loading $envrc" + # shellcheck disable=SC1090,SC1091 + . "$envrc" "$@" + fi +} + +# don't do this it may lead to an infinite loop if 'make link' symlinking ~/.envrc to this repo's .envrc +# (which I do to keep Python virtual automatically loaded at all times because recent pip on Python refuses +# to install to system Python) +#load_if_exists ~/.envrc + +# ============================================================================ # +# P y t h o n +# ============================================================================ # + + #.envrc-aws \ + #.envrc-gcp \ + #.envrc-terraform \ +# shellcheck disable=SC2043 +for envrc in \ + .envrc-python \ + ; do + load_if_exists "$envrc" +done + +# ============================================================================ # +# A W S +# ============================================================================ # + +if [[ "$PWD" =~ /aws/ ]]; then + load_if_exists .envrc-aws +fi + +# ============================================================================ # +# G C P +# ============================================================================ # + +if [[ "$PWD" =~ /gcp/ ]]; then + load_if_exists .envrc-gcp +fi + +# ============================================================================ # +# T e r r a f o r m +# ============================================================================ # + +if [[ "$PWD" =~ /(terra(form)?|tf)(/|$) ]]; then + load_if_exists .envrc-terraform +fi + +# ============================================================================ # +# K u b e r n e t e s +# ============================================================================ # + +if [ -f "$srcdir/.envrc-kubernetes" ]; then + load_if_exists .envrc-kubernetes docker-desktop +fi + +# ============================================================================ # +# . E n v +# ============================================================================ # + +echo +# read .env too +#dotenv + +load_if_exists .envrc.local diff --git a/.envrc-python b/.envrc-python new file mode 100644 index 0000000..039c861 --- /dev/null +++ b/.envrc-python @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: Mon Feb 22 17:42:01 2021 +0000 +# +# https://github.com/HariSekhon/SQL-scripts +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P y t h o n D i r E n v +# ============================================================================ # + +# .envrc to auto-load the virtualenv inside the 'venv' directory if present + +# https://direnv.net/man/direnv-stdlib.1.html + +set -euo pipefail +[ -n "${DEBUG:-}" ] && set -x +#srcdir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# this is necessary because newer versions of pip no longer allow you to install PyPI packages in system-packages by default +for venv in "$PWD/venv" "$HOME/venv"; do + if [ -f "$venv/bin/activate" ]; then + echo + echo "Virtualenv directory found in: $venv" + echo + echo "Activating Virtualenv inside the directory: $venv" + + # shellcheck disable=SC1091 + source "$venv/bin/activate" + break + fi +done + +# read .env too +#dotenv diff --git a/.github/workflows/checkov.yaml b/.github/workflows/checkov.yaml index 5850725..0654613 100644 --- a/.github/workflows/checkov.yaml +++ b/.github/workflows/checkov.yaml @@ -22,7 +22,7 @@ --- name: Checkov -on: +on: # yamllint disable-line rule:truthy push: branches: - master diff --git a/.github/workflows/codeowners.yaml b/.github/workflows/codeowners.yaml index c089f85..071710d 100644 --- a/.github/workflows/codeowners.yaml +++ b/.github/workflows/codeowners.yaml @@ -11,10 +11,14 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# C o d e O w n e r s +# ============================================================================ # + --- -name: Codeowners +name: CodeOwners -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -42,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -52,3 +56,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Validate CODEOWNERS uses: HariSekhon/GitHub-Actions/.github/workflows/codeowners.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/fork-sync.yaml b/.github/workflows/fork-sync.yaml index 28797a0..572275c 100644 --- a/.github/workflows/fork-sync.yaml +++ b/.github/workflows/fork-sync.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k S y n c +# ============================================================================ # + +# For a fork of the original repo, activate to keep it up to date via straight GitHub sync to the default branch + --- name: Fork Sync -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: @@ -28,7 +34,7 @@ permissions: contents: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/fork-update-pr.yaml b/.github/workflows/fork-update-pr.yaml index b6084b0..ca2c7e6 100644 --- a/.github/workflows/fork-update-pr.yaml +++ b/.github/workflows/fork-update-pr.yaml @@ -11,10 +11,18 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# F o r k U p d a t e P R +# ============================================================================ # + +# For a fork of the original repo, activate to keep its branches up to date via Pull Requests +# +# To be used in conjunction with the adjacent fork-sync.yaml which keeps the default branch up to date + --- name: Fork Update PR -on: +on: # yamllint disable-line rule:truthy workflow_dispatch: inputs: debug: @@ -22,14 +30,14 @@ on: required: false default: false schedule: - - cron: '0 10 * * 2' + - cron: '0 10 * * 1' permissions: contents: write pull-requests: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: false jobs: diff --git a/.github/workflows/grype.yaml b/.github/workflows/grype.yaml new file mode 100644 index 0000000..1447493 --- /dev/null +++ b/.github/workflows/grype.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2023-05-13 01:07:56 +0100 (Sat, 13 May 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# G r y p e +# ============================================================================ # + +--- +name: Grype + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + Grype: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Grype + uses: HariSekhon/GitHub-Actions/.github/workflows/grype.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/json.yaml b/.github/workflows/json.yaml index 63da2d4..cdb554f 100644 --- a/.github/workflows/json.yaml +++ b/.github/workflows/json.yaml @@ -11,22 +11,30 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# J S O N +# ============================================================================ # + +# Validate any JSON files found in the repo + --- name: JSON -on: +on: # yamllint disable-line rule:truthy push: branches: - master - main paths: - '**/*.json' + - .github/workflows/json.yaml pull_request: branches: - master - main paths: - '**/*.json' + - .github/workflows/json.yaml workflow_dispatch: inputs: debug: @@ -46,3 +54,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Check JSON uses: HariSekhon/GitHub-Actions/.github/workflows/json.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/kics.yaml b/.github/workflows/kics.yaml new file mode 100644 index 0000000..c3d629c --- /dev/null +++ b/.github/workflows/kics.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-01 19:36:08 +0000 (Tue, 01 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# K i c s +# ============================================================================ # + +--- +name: Kics + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + kics: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Kics + uses: HariSekhon/GitHub-Actions/.github/workflows/kics.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/markdown.yaml b/.github/workflows/markdown.yaml new file mode 100644 index 0000000..7717f8b --- /dev/null +++ b/.github/workflows/markdown.yaml @@ -0,0 +1,54 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# M a r k D o w n +# ============================================================================ # + +--- +name: Markdown + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.md' + - .mdlrc + - .mdl.rb + - .markdownlint.rb + - .github/workflows/markdown.yaml + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + Markdown: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Markdown + uses: HariSekhon/GitHub-Actions/.github/workflows/markdown.yaml@master diff --git a/.github/workflows/semgrep-cloud.yaml b/.github/workflows/semgrep-cloud.yaml index 0304744..d3508c9 100644 --- a/.github/workflows/semgrep-cloud.yaml +++ b/.github/workflows/semgrep-cloud.yaml @@ -20,7 +20,7 @@ --- name: Semgrep Cloud -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -46,7 +46,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -58,3 +58,5 @@ jobs: uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep-cloud.yaml@master secrets: SEMGREP_APP_TOKEN: ${{ secrets.SEMGREP_APP_TOKEN }} + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/semgrep.yaml b/.github/workflows/semgrep.yaml index ae998d9..3005ed1 100644 --- a/.github/workflows/semgrep.yaml +++ b/.github/workflows/semgrep.yaml @@ -22,7 +22,7 @@ --- name: Semgrep -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -50,7 +50,7 @@ permissions: security-events: write concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -60,3 +60,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Semgrep GitHub Security Tab uses: HariSekhon/GitHub-Actions/.github/workflows/semgrep.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/shellcheck.yaml b/.github/workflows/shellcheck.yaml new file mode 100644 index 0000000..642f533 --- /dev/null +++ b/.github/workflows/shellcheck.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S h e l l C h e c k +# ============================================================================ # + +# Validate any shell scripts found in the repo + +--- +name: ShellCheck + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.sh' + - .github/workflows/shellcheck.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.sh' + - .github/workflows/shellcheck.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + shellcheck: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: ShellCheck + uses: HariSekhon/GitHub-Actions/.github/workflows/shellcheck.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/sonarcloud.yaml b/.github/workflows/sonarcloud.yaml new file mode 100644 index 0000000..15cc547 --- /dev/null +++ b/.github/workflows/sonarcloud.yaml @@ -0,0 +1,48 @@ +# +# Author: Hari Sekhon +# Date: 2023-04-14 23:53:43 +0100 (Fri, 14 Apr 2023) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S o n a r C l o u d +# ============================================================================ # + +--- +name: SonarCloud + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + +permissions: + contents: read + pull-requests: read + +jobs: + SonarCloud: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: SonarCloud + uses: HariSekhon/GitHub-Actions/.github/workflows/sonarcloud.yaml@master + secrets: + SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} diff --git a/.github/workflows/sqlfluff.yaml b/.github/workflows/sqlfluff.yaml new file mode 100644 index 0000000..75dde06 --- /dev/null +++ b/.github/workflows/sqlfluff.yaml @@ -0,0 +1,60 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S Q L F l u f f +# ============================================================================ # + +--- +name: SQLFluff + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.sql' + - .github/workflows/sqlfluff.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.sql' + - .github/workflows/sqlfluff.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +concurrency: + group: ${{ github.ref }}-${{ github.workflow }} + cancel-in-progress: true + +jobs: + sqlfluff: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: SQLFluff + uses: HariSekhon/GitHub-Actions/.github/workflows/sqlfluff.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/trivy.yaml b/.github/workflows/trivy.yaml new file mode 100644 index 0000000..74a08f6 --- /dev/null +++ b/.github/workflows/trivy.yaml @@ -0,0 +1,62 @@ +# +# Author: Hari Sekhon +# Date: 2022-02-02 11:27:37 +0000 (Wed, 02 Feb 2022) +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# T r i v y +# ============================================================================ # + +# Scan files in the local repo + +--- +name: Trivy + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths-ignore: + - '**/*.md' + pull_request: + branches: + - master + - main + paths-ignore: + - '**/*.md' + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + actions: read + contents: read + security-events: write + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + trivy: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Trivy + uses: HariSekhon/GitHub-Actions/.github/workflows/trivy.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/validate.yaml b/.github/workflows/validate.yaml index 7dd9421..2122c48 100644 --- a/.github/workflows/validate.yaml +++ b/.github/workflows/validate.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# V a l i d a t i o n +# ============================================================================ # + +# Run all custom validations against files in the repo + --- name: Validation -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -33,14 +39,14 @@ on: type: boolean required: false default: false - schedule: - - cron: '0 0 * * 1' + #schedule: + # - cron: '0 0 * * 1' permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -50,3 +56,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Validate uses: HariSekhon/GitHub-Actions/.github/workflows/validate.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.github/workflows/xml.yaml b/.github/workflows/xml.yaml new file mode 100644 index 0000000..1440b99 --- /dev/null +++ b/.github/workflows/xml.yaml @@ -0,0 +1,56 @@ +# +# Author: Hari Sekhon +# Date: Tue Feb 4 09:53:28 2020 +0000 +# +# vim:ts=2:sts=2:sw=2:et +# +# https://github.com/HariSekhon/SQL-scripts +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# X M L +# ============================================================================ # + +# Validate any XML files found in the repo + +--- +name: XML + +on: # yamllint disable-line rule:truthy + push: + branches: + - master + - main + paths: + - '**/*.xml' + - .github/workflows/xml.yaml + pull_request: + branches: + - master + - main + paths: + - '**/*.xml' + - .github/workflows/xml.yaml + workflow_dispatch: + inputs: + debug: + type: boolean + required: false + default: false + schedule: + - cron: '0 0 * * 1' + +permissions: + contents: read + +jobs: + check_xml: + # github.event.repository context not available in scheduled workflows + #if: github.event.repository.fork == false + if: github.repository_owner == 'HariSekhon' + name: Check XML + uses: HariSekhon/GitHub-Actions/.github/workflows/xml.yaml@master diff --git a/.github/workflows/yaml.yaml b/.github/workflows/yaml.yaml index b980e2c..4a1b8f8 100644 --- a/.github/workflows/yaml.yaml +++ b/.github/workflows/yaml.yaml @@ -11,10 +11,16 @@ # https://www.linkedin.com/in/HariSekhon # +# ============================================================================ # +# Y A M L +# ============================================================================ # + +# Validate any YAML files found in the repo + --- name: YAML -on: +on: # yamllint disable-line rule:truthy push: branches: - master @@ -22,6 +28,7 @@ on: paths: - '**/*.yml' - '**/*.yaml' + - .github/workflows/yaml.yaml pull_request: branches: - master @@ -29,6 +36,7 @@ on: paths: - '**/*.yml' - '**/*.yaml' + - .github/workflows/yaml.yaml workflow_dispatch: inputs: debug: @@ -42,7 +50,7 @@ permissions: contents: read concurrency: - group: ${{ github.ref }}-${{ github.workflow }} + group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true jobs: @@ -52,3 +60,5 @@ jobs: if: github.repository_owner == 'HariSekhon' name: Check YAML uses: HariSekhon/GitHub-Actions/.github/workflows/yaml.yaml@master + with: + debug: ${{ github.event.inputs.debug }} diff --git a/.mdl.rb b/.mdl.rb new file mode 100644 index 0000000..da67a74 --- /dev/null +++ b/.mdl.rb @@ -0,0 +1,31 @@ +#!/usr/bin/env ruby +# vim:ts=4:sts=4:sw=4:et:filetype=ruby +# +# Author: Hari Sekhon +# Date: 2024-08-22 01:58:12 +0200 (Thu, 22 Aug 2024) +# +# https///github.com/HariSekhon/SQL-scripts +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +all +#exclude_rule 'MD001' +#exclude_rule 'MD003' +#exclude_rule 'MD005' +exclude_rule 'MD007' # leave 2 space indentation for lists, 3 space is ugly af +#exclude_rule 'MD012' +exclude_rule 'MD013' # long lines cannot be split if they are URLs +#exclude_rule 'MD022' +#exclude_rule 'MD025' +exclude_rule 'MD026' # Trailing punctuation in header - sometimes I want to do etc. or ... at the end of a heading +#exclude_rule 'MD031' +#exclude_rule 'MD032' +exclude_rule 'MD033' # inline HTML is important for formatting +exclude_rule 'MD036' # emphasis used instead of header for footer Ported from lines +#exclude_rule 'MD039' +#exclude_rule 'MD056' diff --git a/.mdlrc b/.mdlrc new file mode 100644 index 0000000..27e5b68 --- /dev/null +++ b/.mdlrc @@ -0,0 +1,5 @@ +mdlrc_dir = File.expand_path('..', __FILE__) + +style_file = File.join(mdlrc_dir, '.mdl.rb') + +style style_file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..d947e90 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,74 @@ +# +# Author: Hari Sekhon +# Date: 2024-08-08 17:34:56 +0300 (Thu, 08 Aug 2024) +# +# vim:ts=2:sts=2:sw=2:et +# +# https///github.com/HariSekhon/SQL-scripts +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# P r e - C o m m i t +# ============================================================================ # + +--- +fail_fast: false +#exclude: *.tmp$ + +repos: + + # will accept anything that 'git clone' understands + # this means you can set this to a local git repo to develop your own hook repos interactively + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: check-yaml + # Common errors + #- id: end-of-file-fixer # ruins .gitignore Icon\r + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + # Git style + - id: check-added-large-files + - id: check-merge-conflict + - id: check-vcs-permalinks + #- id: forbid-new-submodules + # Cross platform + - id: check-case-conflict + - id: mixed-line-ending + args: [--fix=lf] + # Security + - id: detect-aws-credentials + args: ['--allow-missing-credentials'] + + # rewrites python files with useless changes like changing single quotes to double quotes + #- repo: https://github.com/psf/black + # rev: 24.8.0 + # hooks: + # - id: black + + # Git secrets Leaks + - repo: https://github.com/awslabs/git-secrets.git + # the release tags for 1.2.0, 1.2.1 and 1.3.0 are broken with this error: + # + # /Users/hari/.cache/pre-commit/repo......./.pre-commit-hooks.yaml is not a file + # + rev: 5357e18 + hooks: + - id: git-secrets + + - repo: https://github.com/markdownlint/markdownlint + rev: v0.12.0 + hooks: + - id: markdownlint + name: Markdownlint + description: Run markdownlint on your Markdown files + entry: mdl + args: [-s, .mdl.rb] + language: ruby + files: \.(md|mdown|markdown)$ diff --git a/.sonarlint/connectedMode.json b/.sonarlint/connectedMode.json new file mode 100644 index 0000000..47f535c --- /dev/null +++ b/.sonarlint/connectedMode.json @@ -0,0 +1,4 @@ +{ + "sonarCloudOrganization": "harisekhon", + "projectKey": "HariSekhon_SQL-scripts" +} diff --git a/.travis.yml b/.travis.yml index f2f3891..202277d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,14 +3,14 @@ # Author: Hari Sekhon # Date: 2020-08-12 12:31:25 +0100 (Wed, 12 Aug 2020) # -# https://github.com/harisekhon/sql +# https://github.com/HariSekhon/SQL-scripts # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # to help improve or steer this or other code I publish # -# http://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # ============================================================================ # diff --git a/Makefile b/Makefile index d7ba8e7..5782290 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # On Ubuntu this fails to pushd otherwise diff --git a/README.md b/README.md index b8e876b..c5660d4 100644 --- a/README.md +++ b/README.md @@ -1,45 +1,61 @@ -SQL Scripts -=========== +# SQL Scripts [![GitHub stars](https://img.shields.io/github/stars/HariSekhon/SQL-scripts?logo=github)](https://github.com/HariSekhon/SQL-scripts/stargazers) [![GitHub forks](https://img.shields.io/github/forks/HariSekhon/SQL-scripts?logo=github)](https://github.com/HariSekhon/SQL-scripts/network) -[![Lines of Code](https://img.shields.io/badge/lines%20of%20code-5k-lightgrey?logo=codecademy)](https://github.com/HariSekhon/SQL-scripts#SQL-Scripts) +[![LineCount](https://sloc.xyz/github/HariSekhon/SQL-scripts/?badge-bg-color=2081C2)](https://github.com/boyter/scc/) +[![Cocomo](https://sloc.xyz/github/HariSekhon/SQL-scripts/?badge-bg-color=2081C2&category=cocomo)](https://github.com/boyter/scc/) [![License](https://img.shields.io/github/license/HariSekhon/SQL-scripts)](https://github.com/HariSekhon/SQL-scripts/blob/master/LICENSE) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) [![GitHub Last Commit](https://img.shields.io/github/last-commit/HariSekhon/SQL-scripts?logo=github)](https://github.com/HariSekhon/SQL-scripts/commits/master) -[![PostgreSQL](https://img.shields.io/badge/SQL-PostreSQL-336791?logo=postgresql)](https://www.postgresql.org/) +[![PostgreSQL](https://img.shields.io/badge/SQL-PostreSQL-336791?logo=postgresql&logoColor=white)](https://www.postgresql.org/) [![MySQL](https://img.shields.io/badge/SQL-MySQL-4479A1?logo=mysql&logoColor=white)](https://www.mysql.com/) [![MariaDB](https://img.shields.io/badge/SQL-MariaDB-003545?logo=mariadb)](https://mariadb.org/) -[![AWS Athena](https://img.shields.io/badge/SQL-AWS%20Athena-232F3E?logo=amazon%20aws)](https://aws.amazon.com/athena/) -[![AWS Aurora](https://img.shields.io/badge/SQL-AWS%20Aurora-232F3E?logo=amazon%20aws)](https://aws.amazon.com/aurora/) -[![Google BigQuery](https://img.shields.io/badge/SQL-Google%20BigQuery-4285F4?logo=google%20cloud)](https://cloud.google.com/bigquery) +[![Oracle](https://img.shields.io/badge/SQL-Oracle-F80000?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+T3JhY2xlPC90aXRsZT48cGF0aCBkPSJNMTYuNDEyIDQuNDEyaC04LjgyYTcuNTg4IDcuNTg4IDAgMCAwLS4wMDggMTUuMTc2aDguODI4YTcuNTg4IDcuNTg4IDAgMCAwIDAtMTUuMTc2em0tLjE5MyAxMi41MDJINy43ODZhNC45MTUgNC45MTUgMCAwIDEgMC05LjgyOGg4LjQzM2E0LjkxNCA0LjkxNCAwIDEgMSAwIDkuODI4eiIgZmlsbD0iI2ZmZmZmZiIgLz48L3N2Zz4K&logoColor=white)](https://oracle.com/) +[![AWS Athena](https://img.shields.io/badge/SQL-AWS%20Athena-232F3E?logo=amazonwebservices)](https://aws.amazon.com/athena/) +[![AWS Aurora](https://img.shields.io/badge/SQL-AWS%20Aurora-232F3E?logo=amazonwebservices)](https://aws.amazon.com/aurora/) +[![Google BigQuery](https://img.shields.io/badge/SQL-Google%20BigQuery-4285F4?logo=google%20cloud&logoColor=white)](https://cloud.google.com/bigquery) + +[![Codacy](https://app.codacy.com/project/badge/Grade/c7ac78789d854b12aa3d23f36953f7e7)](https://www.codacy.com/gh/HariSekhon/SQL-scripts/dashboard) +[![CodeFactor](https://www.codefactor.io/repository/github/harisekhon/SQL-scripts/badge)](https://www.codefactor.io/repository/github/harisekhon/SQL-scripts) +[![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_SQL-scripts&metric=alert_status)](https://sonarcloud.io/dashboard?id=HariSekhon_SQL-scripts) +[![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_SQL-scripts&metric=sqale_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_SQL-scripts) +[![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_SQL-scripts&metric=reliability_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_SQL-scripts) +[![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_SQL-scripts&metric=security_rating)](https://sonarcloud.io/dashboard?id=HariSekhon_SQL-scripts) +[![Vulnerabilities](https://sonarcloud.io/api/project_badges/measure?project=HariSekhon_SQL-scripts&metric=vulnerabilities)](https://sonarcloud.io/summary/new_code?id=HariSekhon_SQL-scripts) [![CI Builds Overview](https://img.shields.io/badge/CI%20Builds-Overview%20Page-blue?logo=circleci)](https://bitbucket.org/HariSekhon/devops-bash-tools/src/master/STATUS.md) +[![ShellCheck](https://github.com/HariSekhon/SQL-scripts/actions/workflows/shellcheck.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/shellcheck.yaml) +[![YAML](https://github.com/HariSekhon/SQL-scripts/actions/workflows/yaml.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/yaml.yaml) +[![Markdown](https://github.com/HariSekhon/SQL-scripts/actions/workflows/markdown.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/markdown.yaml) [![Validation](https://github.com/HariSekhon/SQL-scripts/actions/workflows/validate.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/validate.yaml) +[![Grype](https://github.com/HariSekhon/SQL-scripts/actions/workflows/grype.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/grype.yaml) +[![Kics](https://github.com/HariSekhon/SQL-scripts/actions/workflows/kics.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/kics.yaml) [![Semgrep](https://github.com/HariSekhon/SQL-scripts/actions/workflows/semgrep.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/semgrep.yaml) -[![Travis CI](https://img.shields.io/badge/TravisCI-ready-blue?logo=travis&label=Travis%20CI)](https://github.com/HariSekhon/SQL-scripts/blob/master/.travis.yml) +[![Semgrep Cloud](https://github.com/HariSekhon/SQL-scripts/actions/workflows/semgrep-cloud.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/semgrep-cloud.yaml) +[![Trivy](https://github.com/HariSekhon/SQL-scripts/actions/workflows/trivy.yaml/badge.svg)](https://github.com/HariSekhon/SQL-scripts/actions/workflows/trivy.yaml) [![Linux](https://img.shields.io/badge/OS-Linux-blue?logo=linux)](https://github.com/HariSekhon/SQL-scripts#SQL-Scripts) [![Mac](https://img.shields.io/badge/OS-Mac-blue?logo=apple)](https://github.com/HariSekhon/SQL-scripts#SQL-Scripts) -[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/SQL-scripts) [![Repo on GitHub](https://img.shields.io/badge/repo-GitHub-2088FF?logo=github)](https://github.com/HariSekhon/SQL-scripts) [![Repo on GitLab](https://img.shields.io/badge/repo-GitLab-FCA121?logo=gitlab)](https://gitlab.com/HariSekhon/SQL-scripts) +[![Repo on Azure DevOps](https://img.shields.io/badge/repo-Azure%20DevOps-0078D7?logo=azure%20devops)](https://dev.azure.com/harisekhon/GitHub/_git/SQL-scripts) [![Repo on BitBucket](https://img.shields.io/badge/repo-BitBucket-0052CC?logo=bitbucket)](https://bitbucket.org/HariSekhon/SQL-scripts) [git.io/SQL](https://git.io/SQL) -Useful SQL scripts, split from [DevOps Bash tools](https://github.com/harisekhon/devops-bash-tools), for which this is now a submodule. +Useful SQL scripts, split from [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools), for which this is now a submodule. Hari Sekhon Cloud & Big Data Contractor, United Kingdom -[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=linkedin)](https://www.linkedin.com/in/harisekhon/) -###### (you're welcome to connect with me on LinkedIn) +[![My LinkedIn](https://img.shields.io/badge/LinkedIn%20Profile-HariSekhon-blue?logo=data:image/svg%2bxml;base64,PHN2ZyByb2xlPSJpbWciIGZpbGw9IiNmZmZmZmYiIHZpZXdCb3g9IjAgMCAyNCAyNCIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj48dGl0bGU+TGlua2VkSW48L3RpdGxlPjxwYXRoIGQ9Ik0yMC40NDcgMjAuNDUyaC0zLjU1NHYtNS41NjljMC0xLjMyOC0uMDI3LTMuMDM3LTEuODUyLTMuMDM3LTEuODUzIDAtMi4xMzYgMS40NDUtMi4xMzYgMi45Mzl2NS42NjdIOS4zNTFWOWgzLjQxNHYxLjU2MWguMDQ2Yy40NzctLjkgMS42MzctMS44NSAzLjM3LTEuODUgMy42MDEgMCA0LjI2NyAyLjM3IDQuMjY3IDUuNDU1djYuMjg2ek01LjMzNyA3LjQzM2MtMS4xNDQgMC0yLjA2My0uOTI2LTIuMDYzLTIuMDY1IDAtMS4xMzguOTItMi4wNjMgMi4wNjMtMi4wNjMgMS4xNCAwIDIuMDY0LjkyNSAyLjA2NCAyLjA2MyAwIDEuMTM5LS45MjUgMi4wNjUtMi4wNjQgMi4wNjV6bTEuNzgyIDEzLjAxOUgzLjU1NVY5aDMuNTY0djExLjQ1MnpNMjIuMjI1IDBIMS43NzFDLjc5MiAwIDAgLjc3NCAwIDEuNzI5djIwLjU0MkMwIDIzLjIyNy43OTIgMjQgMS43NzEgMjRoMjAuNDUxQzIzLjIgMjQgMjQgMjMuMjI3IDI0IDIyLjI3MVYxLjcyOUMyNCAuNzc0IDIzLjIgMCAyMi4yMjIgMGguMDAzeiIvPjwvc3ZnPgo=)](https://www.linkedin.com/in/HariSekhon/) +
*(you're welcome to connect with me on LinkedIn)* -### Inventory +## Inventory -#### DevOps / DBA +### DevOps / DBA - `aws_athena_cloudtrail_ddl.sql` - [AWS Athena](https://aws.amazon.com/athena/) DDL to setup up integration to query [CloudTrail](https://aws.amazon.com/cloudtrail/) logs from Athena - `bigquery_*.sql` - [Google BigQuery](https://cloud.google.com/bigquery) scripts: @@ -54,8 +70,11 @@ Cloud & Big Data Contractor, United Kingdom - [PostgreSQL](https://www.postgresql.org/) queries for DBA investigating + performance tuning - [postgres_info.sql](https://github.com/HariSekhon/SQL-scripts/blob/master/postgres_info.sql) - big summary overview, recommend you start here - tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.x +- `oracle_*.sql`: + - [Oracle](https://www.oracle.com) queries for DBA investigating + - tested on Oracle 9i, 10g, 11g, 19c -#### Analytics +### Analytics - `bigquery_*.sql` - [Google BigQuery](https://cloud.google.com/bigquery) scripts: - `bigquery_billing_*.sql` - billing queries for [GCP](https://cloud.google.com/) usage eg. highest cost services, most used GCP products, recent charges etc. @@ -63,65 +82,150 @@ Cloud & Big Data Contractor, United Kingdom - [analytics/](https://github.com/HariSekhon/SQL-scripts/tree/master/analytics)`bigquery_*.sql` - ecommerce queries and [BigQuery ML](https://cloud.google.com/bigquery-ml/docs/bigqueryml-intro) machine learning classification logistic regression models and purchasing predictions - for more [BigQuery](https://cloud.google.com/bigquery) examples, see [Data Engineering demos](https://github.com/GoogleCloudPlatform/training-data-analyst/tree/master/courses/data-engineering/demos) -#### DevOps SQL tooling +### Database Knowledge Base + +See the pages for: + +- [SQL](https://github.com/HariSekhon/Knowledge-Base/blob/main/sql.md) +- [SQL Databases](https://github.com/HariSekhon/Knowledge-Base/blob/main/databases.md) +- [MySQL](https://github.com/HariSekhon/Knowledge-Base/blob/main/mysql.md) +- [PostgreSQL](https://github.com/HariSekhon/Knowledge-Base/blob/main/postgres.md) +- [Oracle](https://github.com/HariSekhon/Knowledge-Base/blob/main/oracle.md) + +in the [HariSekhon/Knowledge-Base](https://github.com/HariSekhon/Knowledge-Base) repo: + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) -You can quickly test the PostgreSQL / MySQL scripts using `postgres.sh` / `mysqld.sh` / `mariadb.sh` in the [DevOps Bash tools](https://github.com/harisekhon/devops-bash-tools) repo, which boots a docker container and drops straight in to a `mysql` / `psql` shell with this directory mounted at `/sql` and used as `$PWD` for fast easy sourcing eg. +### DevOps SQL tooling + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) + +You can quickly test the PostgreSQL / MySQL scripts using `postgres.sh` / `mysqld.sh` / `mariadb.sh` in the [DevOps Bash tools](https://github.com/HariSekhon/DevOps-Bash-tools) repo, which boots a docker container and drops straight in to a `mysql` / `psql` shell with this directory mounted at `/sql` and used as `$PWD` for fast easy sourcing eg. postgres: -``` + +```postgres-sql \i /sql/postgres_query_times.sql ``` -``` + +```postgres-sql \i postgres_query_times.sql ``` mysql: -``` + +```mysql-sql source /sql/mysql_sessions.sql ``` -``` + +```mysql-sql \. mysql_sessions.sql ``` - -#### Related scripts +### Related scripts - [.psqlrc](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.psqlrc) - advanced PostgreSQL psql client config - [psql.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/psql.sh) - quickly connect to PostgreSQL with command line switches inferred from environment variables - [mysql.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/mysql.sh) - quickly connect to MySQL / MariaDB with command line switches inferred from environment variables - [postgres.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/postgres.sh) - one-touch PostgreSQL, boots docker container and drops you in to `psql` shell. Version can be given as an argument - [mysqld.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/mysqld.sh) / [mariadb.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/mariadb.sh) - one-touch MySQL / MariaDB, boots docker container and drops you in to `mysql` shell. Version can be given as an argument -- [postgres_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/postgres_foreach_table.sh) / [mysql_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/mysql_foreach_table.sh) - execute templated SQL queries/statements against all or a subset of tables -- [postgres_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/postgres_tables_row_counts.sh) / [mysql_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/mysql_tables_row_counts.sh) - get row counts for all or a subset of tables +- [postgres_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/postgres/postgres_foreach_table.sh) / [mysql_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/mysql/mysql_foreach_table.sh) - execute templated SQL queries/statements against all or a subset of tables +- [postgres_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/postgres/postgres_tables_row_counts.sh) / [mysql_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/mysql/mysql_tables_row_counts.sh) - get row counts for all or a subset of tables - [sqlcase.pl](https://github.com/HariSekhon/DevOps-Perl-tools/blob/master/sqlcase.pl) - autocases your SQL code - - I use this a lot and call it via hotkey configured in my [.vimrc](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/.vimrc) + - I use this a lot and call it via hotkey configured in my [.vimrc](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/configs/.vimrc) - there are `*case.pl` specializations for most of the major RDBMS and distributed SQL systems, even several NoSQL systems, using each one's language specific keywords - Hive & Impala SQL: - - [beeline.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/beeline.sh) - quickly connect to Hive, auto-determines HiveServer2 address, Kerberos & SSL options, ZooKeeper quorum - - [impala_shell.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/impala_shell.sh) - quickly connect to Impala, auto-determines a Hadoop worker node address and Kerberos options (can use an environment variable for a [Load Balancer](https://github.com/HariSekhon/HAProxy-configs/blob/master/impala-jdbc.cfg) setup) - - [hive_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/hive_foreach_table.sh) / [impala_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/impala_foreach_table.sh) - execute templated SQL queries/statements against all or a subset of tables - - [hive_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/hive_tables_row_counts.sh) / [impala_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/impala_tables_row_counts.sh) - get row counts for all or a subset of tables - - [hive_tables_column_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/hive_tables_column_counts.sh) / [impala_tables_column_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/impala_tables_column_counts.sh) - get the column counts for big tables in Hive / Impala - - [hive_tables_metadata.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/hive_tables_metadata.sh) / [impala_tables_metadata.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/impala_tables_metadata.sh) / [hive_tables_locations.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/hive_tables_locations.sh) / [impala_tables_locations.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/impala_tables_locations.sh) - get Hive / Impala metadata for all or a subset of tables, eg. Location to determine where the external tables data is being stored (HDFS / S3 paths) + - [beeline.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/beeline.sh) - quickly connect to Hive, auto-determines HiveServer2 address, Kerberos & SSL options, ZooKeeper quorum + - [impala_shell.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/impala_shell.sh) - quickly connect to Impala, auto-determines a Hadoop worker node address and Kerberos options (can use an environment variable for a [Load Balancer](https://github.com/HariSekhon/HAProxy-configs/blob/master/bigdata/impala-jdbc.cfg) setup) + - [hive_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/hive_foreach_table.sh) / [impala_foreach_table.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/impala_foreach_table.sh) - execute templated SQL queries/statements against all or a subset of tables + - [hive_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/hive_tables_row_counts.sh) / [impala_tables_row_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/impala_tables_row_counts.sh) - get row counts for all or a subset of tables + - [hive_tables_column_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/hive_tables_column_counts.sh) / [impala_tables_column_counts.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/impala_tables_column_counts.sh) - get the column counts for big tables in Hive / Impala + - [hive_tables_metadata.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/hive_tables_metadata.sh) / [impala_tables_metadata.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/impala_tables_metadata.sh) / [hive_tables_locations.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/hive_tables_locations.sh) / [impala_tables_locations.sh](https://github.com/HariSekhon/DevOps-Bash-tools/blob/master/bigdata/impala_tables_locations.sh) - get Hive / Impala metadata for all or a subset of tables, eg. Location to determine where the external tables data is being stored (HDFS / S3 paths) -### See Also +## Star History -- [DevOps Bash Tools](https://github.com/harisekhon/devops-bash-tools) - 700+ DevOps Bash Scripts, Advanced `.bashrc`, `.vimrc`, `.screenrc`, `.tmux.conf`, `.gitconfig`, CI configs & Utility Code Library - AWS, GCP, Kubernetes, Docker, Kafka, Hadoop, SQL, BigQuery, Hive, Impala, PostgreSQL, MySQL, LDAP, DockerHub, Jenkins, Spotify API & MP3 tools, Git tricks, GitHub API, GitLab API, BitBucket API, Code & build linting, package management for Linux / Mac / Python / Perl / Ruby / NodeJS / Golang, and lots more random goodies +[![Star History Chart](https://api.star-history.com/svg?repos=HariSekhon/SQL-scripts&type=Date)](https://star-history.com/#HariSekhon/SQL-scripts&Date) -- [Templates](https://github.com/HariSekhon/Templates) - dozens of Code & Config templates - AWS, GCP, Docker, Jenkins, Terraform, Vagrant, Puppet, Python, Bash, Go, Perl, Java, Scala, Groovy, Maven, SBT, Gradle, Make, GitHub Actions Workflows, CircleCI, Jenkinsfile, Makefile, Dockerfile, docker-compose.yml, M4 etc. +[git.io/SQL](https://git.io/SQL) -- [Kubernetes configs](https://github.com/HariSekhon/Kubernetes-configs) - Kubernetes YAML configs - Best Practices, Tips & Tricks are baked right into the templates for future deployments +## More Core Repos -- [DevOps Python Tools](https://github.com/harisekhon/devops-python-tools) - 80+ DevOps CLI tools for AWS, GCP, Hadoop, HBase, Spark, Log Anonymizer, Ambari Blueprints, AWS CloudFormation, Linux, Docker, Spark Data Converters & Validators (Avro / Parquet / JSON / CSV / INI / XML / YAML), Elasticsearch, Solr, Travis CI, Pig, IPython + -- [The Advanced Nagios Plugins Collection](https://github.com/harisekhon/nagios-plugins) - 450+ programs for Nagios monitoring your Hadoop & NoSQL clusters. Covers every Hadoop vendor's management API and every major NoSQL technology (HBase, Cassandra, MongoDB, Elasticsearch, Solr, Riak, Redis etc.) as well as message queues (Kafka, RabbitMQ), continuous integration (Jenkins, Travis CI) and traditional infrastructure (SSL, Whois, DNS, Linux) +### Knowledge -- [DevOps Perl Tools](https://github.com/harisekhon/perl-tools) - 25+ DevOps CLI tools for Hadoop, HDFS, Hive, Solr/SolrCloud CLI, Log Anonymizer, Nginx stats & HTTP(S) URL watchers for load balanced web farms, Dockerfiles & SQL ReCaser (MySQL, PostgreSQL, AWS Redshift, Snowflake, Apache Drill, Hive, Impala, Cassandra CQL, Microsoft SQL Server, Oracle, Couchbase N1QL, Dockerfiles, Pig Latin, Neo4j, InfluxDB), Ambari FreeIPA Kerberos, Datameer, Linux... +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Knowledge-Base&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Knowledge-Base) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Diagrams-as-Code&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Diagrams-as-Code) -- [HAProxy Configs](https://github.com/HariSekhon/HAProxy-configs) - 80+ HAProxy Configs for Hadoop, Big Data, NoSQL, Docker, Elasticsearch, SolrCloud, HBase, Cloudera, Hortonworks, MapR, MySQL, PostgreSQL, Apache Drill, Hive, Presto, Impala, ZooKeeper, OpenTSDB, InfluxDB, Prometheus, Kibana, Graphite, SSH, RabbitMQ, Redis, Riak, Rancher etc. + + +### DevOps Code + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Bash-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Bash-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Python-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Python-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Perl-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Perl-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=DevOps-Golang-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/DevOps-Golang-tools) + + + +### Containerization + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Kubernetes-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Kubernetes-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Dockerfiles&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Dockerfiles) + +### CI/CD + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=GitHub-Actions&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/GitHub-Actions) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Jenkins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Jenkins) + +### Databases - DBA - SQL + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=SQL-scripts&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/SQL-scripts) + +### DevOps Reloaded + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=HAProxy-configs&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/HAProxy-configs) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Terraform&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Terraform) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Packer&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Packer) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Ansible&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Ansible) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Environments&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Environments) + +### Monitoring + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugins&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugins) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Nagios-Plugin-Kafka&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Nagios-Plugin-Kafka) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Prometheus&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Prometheus) + +### Templates + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Templates&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Templates) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Template-repo&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Template-repo) + +### Desktop + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=TamperMonkey&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/TamperMonkey) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Hammerspoon&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Hammerspoon) + +### Spotify + +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-tools&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-tools) +[![Readme Card](https://github-readme-stats.vercel.app/api/pin/?username=HariSekhon&repo=Spotify-playlists&theme=ambient_gradient&description_lines_count=3)](https://github.com/HariSekhon/Spotify-playlists) + +The rest of my original source repos are +[here](https://github.com/HariSekhon?tab=repositories&q=&type=source&language=&sort=stargazers). + +Pre-built Docker images are available on my [DockerHub](https://hub.docker.com/u/harisekhon/). + + diff --git a/analytics/README.md b/analytics/README.md index 9ded54f..163d575 100644 --- a/analytics/README.md +++ b/analytics/README.md @@ -4,6 +4,6 @@ Load public dataset into your BigQuery using this link: -https://console.cloud.google.com/bigquery?p=data-to-insights&d=ecommerce&t=web_analytics&page=table + Then run the analytics SQL examples from here. diff --git a/analytics/bigquery_ecommerce_conversion_rate.sql b/analytics/bigquery_ecommerce_conversion_rate.sql index 030c677..4f7cfe8 100644 --- a/analytics/bigquery_ecommerce_conversion_rate.sql +++ b/analytics/bigquery_ecommerce_conversion_rate.sql @@ -2,36 +2,36 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 09:22:11 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- WITH visitors AS( - SELECT - COUNT(DISTINCT fullVisitorId) AS total_visitors - FROM `data-to-insights.ecommerce.web_analytics` + SELECT + COUNT(DISTINCT fullVisitorId) AS total_visitors + FROM `data-to-insights.ecommerce.web_analytics` ), purchasers AS( - SELECT - COUNT(DISTINCT fullVisitorId) AS total_purchasers - FROM `data-to-insights.ecommerce.web_analytics` - WHERE totals.transactions IS NOT NULL + SELECT + COUNT(DISTINCT fullVisitorId) AS total_purchasers + FROM `data-to-insights.ecommerce.web_analytics` + WHERE totals.transactions IS NOT NULL ) SELECT - total_visitors, - total_purchasers, - total_purchasers / total_visitors AS conversion_rate, - ROUND(total_purchasers / total_visitors * 100, 2) AS conversion_percentage + total_visitors, + total_purchasers, + total_purchasers / total_visitors AS conversion_rate, + ROUND(total_purchasers / total_visitors * 100, 2) AS conversion_percentage FROM - visitors, - purchasers + visitors, + purchasers ; diff --git a/analytics/bigquery_ecommerce_purchasers_on_return_visit.sql b/analytics/bigquery_ecommerce_purchasers_on_return_visit.sql index 7cfe2dd..953db5a 100644 --- a/analytics/bigquery_ecommerce_purchasers_on_return_visit.sql +++ b/analytics/bigquery_ecommerce_purchasers_on_return_visit.sql @@ -2,36 +2,36 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 09:39:05 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- visitors who bought on a return visit (could have bought on first as well) WITH all_visitor_stats AS ( SELECT - fullvisitorid, - IF( - COUNTIF(totals.transactions > 0 - AND - totals.newVisits IS NULL) > 0, 1, 0 + fullvisitorid, + IF( + COUNTIF(totals.transactions > 0 + AND + totals.newVisits IS NULL) > 0, 1, 0 ) AS will_buy_on_return_visit - FROM - `data-to-insights.ecommerce.web_analytics` - GROUP BY - fullvisitorid + FROM + `data-to-insights.ecommerce.web_analytics` + GROUP BY + fullvisitorid ) SELECT - COUNT(DISTINCT fullvisitorid) AS total_visitors, - will_buy_on_return_visit + COUNT(DISTINCT fullvisitorid) AS total_visitors, + will_buy_on_return_visit FROM - all_visitor_stats + all_visitor_stats GROUP BY - will_buy_on_return_visit; + will_buy_on_return_visit; diff --git a/analytics/bigquery_ecommerce_time_on_site.sql b/analytics/bigquery_ecommerce_time_on_site.sql index 67c24f9..af4401f 100644 --- a/analytics/bigquery_ecommerce_time_on_site.sql +++ b/analytics/bigquery_ecommerce_time_on_site.sql @@ -2,46 +2,46 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 09:49:38 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- looking at bounces and time on site are not good features for model training and prediction since of the top 10 customers by time on site only 1 purchased! SELECT - * EXCEPT(fullVisitorId) + * EXCEPT(fullVisitorId) FROM - # features - (SELECT - fullVisitorId, - IFNULL(totals.bounces, 0) AS bounces, - IFNULL(totals.timeOnSite, 0) AS time_on_site - FROM - `data-to-insights.ecommerce.web_analytics` - WHERE - totals.newVisits = 1) - JOIN - (SELECT - fullvisitorid, - IF( - COUNTIF( - totals.transactions > 0 - AND - totals.newVisits IS NULL - ) > 0, 1, 0 - ) AS will_buy_on_return_visit - FROM - `data-to-insights.ecommerce.web_analytics` - GROUP BY fullvisitorid) - USING (fullVisitorId) + # features + (SELECT + fullVisitorId, + IFNULL(totals.bounces, 0) AS bounces, + IFNULL(totals.timeOnSite, 0) AS time_on_site + FROM + `data-to-insights.ecommerce.web_analytics` + WHERE + totals.newVisits = 1) + JOIN + (SELECT + fullvisitorid, + IF( + COUNTIF( + totals.transactions > 0 + AND + totals.newVisits IS NULL + ) > 0, 1, 0 + ) AS will_buy_on_return_visit + FROM + `data-to-insights.ecommerce.web_analytics` + GROUP BY fullvisitorid) + USING (fullVisitorId) ORDER BY - time_on_site DESC + time_on_site DESC LIMIT 10; diff --git a/analytics/bigquery_ecommerce_top_5_products.sql b/analytics/bigquery_ecommerce_top_5_products.sql index 0aa174d..269bd5b 100644 --- a/analytics/bigquery_ecommerce_top_5_products.sql +++ b/analytics/bigquery_ecommerce_top_5_products.sql @@ -2,27 +2,27 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 09:36:31 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- SELECT - p.v2ProductName, - p.v2ProductCategory, - SUM(p.productQuantity) AS units_sold, - ROUND(SUM(p.localProductRevenue/1000000),2) AS revenue + p.v2ProductName, + p.v2ProductCategory, + SUM(p.productQuantity) AS units_sold, + ROUND(SUM(p.localProductRevenue/1000000),2) AS revenue FROM - `data-to-insights.ecommerce.web_analytics`, + `data-to-insights.ecommerce.web_analytics`, UNNEST(hits) AS h, UNNEST(h.product) AS p GROUP BY 1, 2 ORDER BY - revenue DESC + revenue DESC LIMIT 5; diff --git a/analytics/bigquery_ml_classification_logistic_regression_create_model1.sql b/analytics/bigquery_ml_classification_logistic_regression_create_model1.sql index e233e0b..a1da965 100644 --- a/analytics/bigquery_ml_classification_logistic_regression_create_model1.sql +++ b/analytics/bigquery_ml_classification_logistic_regression_create_model1.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 09:54:10 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- ran for 2 minutes to build the trained model @@ -23,40 +23,40 @@ CREATE OR REPLACE MODEL `ecommerce.classification_model1` OPTIONS ( - model_type='logistic_reg', - labels = ['will_buy_on_return_visit'] + model_type='logistic_reg', + labels = ['will_buy_on_return_visit'] ) AS -- training data provided below via SELECT statement SELECT - * EXCEPT(fullVisitorId) + * EXCEPT(fullVisitorId) FROM - -- features - ( - SELECT - fullVisitorId, - IFNULL(totals.bounces, 0) AS bounces, - IFNULL(totals.timeOnSite, 0) AS time_on_site - FROM - `data-to-insights.ecommerce.web_analytics` - WHERE - totals.newVisits = 1 - -- train on subset of the data so we can test on the rest of the data - AND date BETWEEN '20160801' AND '20170430' - ) - JOIN - ( - SELECT - fullvisitorid, - -- the label of answers to train on - IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit - FROM - `data-to-insights.ecommerce.web_analytics` - GROUP BY - fullvisitorid - ) - USING - (fullVisitorId) + -- features + ( + SELECT + fullVisitorId, + IFNULL(totals.bounces, 0) AS bounces, + IFNULL(totals.timeOnSite, 0) AS time_on_site + FROM + `data-to-insights.ecommerce.web_analytics` + WHERE + totals.newVisits = 1 + -- train on subset of the data so we can test on the rest of the data + AND date BETWEEN '20160801' AND '20170430' + ) + JOIN + ( + SELECT + fullvisitorid, + -- the label of answers to train on + IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit + FROM + `data-to-insights.ecommerce.web_analytics` + GROUP BY + fullvisitorid + ) + USING + (fullVisitorId) ; diff --git a/analytics/bigquery_ml_classification_logistic_regression_create_model2.sql b/analytics/bigquery_ml_classification_logistic_regression_create_model2.sql index e5cefb5..b7d75b3 100644 --- a/analytics/bigquery_ml_classification_logistic_regression_create_model2.sql +++ b/analytics/bigquery_ml_classification_logistic_regression_create_model2.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 10:02:02 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- ran in 2 mins @@ -19,70 +19,70 @@ CREATE OR REPLACE MODEL `ecommerce.classification_model_2` OPTIONS ( - model_type='logistic_reg', - labels = ['will_buy_on_return_visit'] + model_type='logistic_reg', + labels = ['will_buy_on_return_visit'] ) AS -- training data provided below via SELECT statements WITH all_visitor_stats AS ( - SELECT - fullvisitorid, - IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit - FROM `data-to-insights.ecommerce.web_analytics` - GROUP BY fullvisitorid + SELECT + fullvisitorid, + IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit + FROM `data-to-insights.ecommerce.web_analytics` + GROUP BY fullvisitorid ) -- select features SELECT * EXCEPT(unique_session_id) FROM ( - SELECT - CONCAT(fullvisitorid, CAST(visitId AS STRING)) AS unique_session_id, - - -- labels - will_buy_on_return_visit, - - MAX(CAST(h.eCommerceAction.action_type AS INT64)) AS latest_ecommerce_progress, - - -- behavior on the site - IFNULL(totals.bounces, 0) AS bounces, - IFNULL(totals.timeOnSite, 0) AS time_on_site, - totals.pageviews, - - -- where the visitor came from - trafficSource.source, - trafficSource.medium, - channelGrouping, - - -- mobile or desktop - device.deviceCategory, - - -- geographic - IFNULL(geoNetwork.country, "") AS country - - FROM `data-to-insights.ecommerce.web_analytics`, - UNNEST(hits) AS h - - JOIN - all_visitor_stats - USING - (fullvisitorid) - - WHERE 1=1 - -- only predict for new visits - AND totals.newVisits = 1 - -- train on subset of the data so we can test on the rest of the data - AND date BETWEEN '20160801' AND '20170430' - - GROUP BY - unique_session_id, - will_buy_on_return_visit, - bounces, - time_on_site, - totals.pageviews, - trafficSource.source, - trafficSource.medium, - channelGrouping, - device.deviceCategory, - country + SELECT + CONCAT(fullvisitorid, CAST(visitId AS STRING)) AS unique_session_id, + + -- labels + will_buy_on_return_visit, + + MAX(CAST(h.eCommerceAction.action_type AS INT64)) AS latest_ecommerce_progress, + + -- behavior on the site + IFNULL(totals.bounces, 0) AS bounces, + IFNULL(totals.timeOnSite, 0) AS time_on_site, + totals.pageviews, + + -- where the visitor came from + trafficSource.source, + trafficSource.medium, + channelGrouping, + + -- mobile or desktop + device.deviceCategory, + + -- geographic + IFNULL(geoNetwork.country, "") AS country + + FROM `data-to-insights.ecommerce.web_analytics`, + UNNEST(hits) AS h + + JOIN + all_visitor_stats + USING + (fullvisitorid) + + WHERE 1=1 + -- only predict for new visits + AND totals.newVisits = 1 + -- train on subset of the data so we can test on the rest of the data + AND date BETWEEN '20160801' AND '20170430' + + GROUP BY + unique_session_id, + will_buy_on_return_visit, + bounces, + time_on_site, + totals.pageviews, + trafficSource.source, + trafficSource.medium, + channelGrouping, + device.deviceCategory, + country ); diff --git a/analytics/bigquery_ml_classification_logistic_regression_evaluate_model1.sql b/analytics/bigquery_ml_classification_logistic_regression_evaluate_model1.sql index 2d34c8a..8f5bb7f 100644 --- a/analytics/bigquery_ml_classification_logistic_regression_evaluate_model1.sql +++ b/analytics/bigquery_ml_classification_logistic_regression_evaluate_model1.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 09:59:49 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- evaluate model1 @@ -20,48 +20,48 @@ -- poor ROC auc only ~ 0.7x score - go to model 2 SELECT - roc_auc, - CASE - WHEN roc_auc > .9 THEN 'good' - WHEN roc_auc > .8 THEN 'fair' - WHEN roc_auc > .7 THEN 'not great' - ELSE 'poor' END AS model_quality, - accuracy, - precision, - recall + roc_auc, + CASE + WHEN roc_auc > .9 THEN 'good' + WHEN roc_auc > .8 THEN 'fair' + WHEN roc_auc > .7 THEN 'not great' + ELSE 'poor' END AS model_quality, + accuracy, + precision, + recall FROM - ML.EVALUATE( - MODEL ecommerce.classification_model1, ( + ML.EVALUATE( + MODEL ecommerce.classification_model1, ( - -- the test data is supplied below via a SELECT query for the model to compare its predictions to (notice this is non-overlapping with the training data) + -- the test data is supplied below via a SELECT query for the model to compare its predictions to (notice this is non-overlapping with the training data) - SELECT - * EXCEPT(fullVisitorId) - FROM - - -- features - ( SELECT - fullVisitorId, - IFNULL(totals.bounces, 0) AS bounces, - IFNULL(totals.timeOnSite, 0) AS time_on_site + * EXCEPT(fullVisitorId) FROM - `data-to-insights.ecommerce.web_analytics` - WHERE - totals.newVisits = 1 - -- notice this time period doesn't overlap with the training data - AND date BETWEEN '20170501' AND '20170630' - ) - JOIN - ( - SELECT - fullvisitorid, - IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit - FROM - `data-to-insights.ecommerce.web_analytics` - GROUP BY - fullvisitorid + + -- features + ( + SELECT + fullVisitorId, + IFNULL(totals.bounces, 0) AS bounces, + IFNULL(totals.timeOnSite, 0) AS time_on_site + FROM + `data-to-insights.ecommerce.web_analytics` + WHERE + totals.newVisits = 1 + -- notice this time period doesn't overlap with the training data + AND date BETWEEN '20170501' AND '20170630' + ) + JOIN + ( + SELECT + fullvisitorid, + IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit + FROM + `data-to-insights.ecommerce.web_analytics` + GROUP BY + fullvisitorid + ) + USING (fullVisitorId) ) - USING (fullVisitorId) - ) - ); + ); diff --git a/analytics/bigquery_ml_classification_logistic_regression_evaluate_model2.sql b/analytics/bigquery_ml_classification_logistic_regression_evaluate_model2.sql index d044266..15c2022 100644 --- a/analytics/bigquery_ml_classification_logistic_regression_evaluate_model2.sql +++ b/analytics/bigquery_ml_classification_logistic_regression_evaluate_model2.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 10:03:32 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- evaluate model2 with more features @@ -20,73 +20,77 @@ -- good ROC auc > 0.9 - gives much better predictive power than model1 SELECT - roc_auc, - CASE - WHEN roc_auc > .9 THEN 'good' - WHEN roc_auc > .8 THEN 'fair' - WHEN roc_auc > .7 THEN 'not great' - ELSE 'poor' END AS model_quality, - accuracy, - precision, - recall + roc_auc, + CASE + WHEN roc_auc > .9 THEN 'good' + WHEN roc_auc > .8 THEN 'fair' + WHEN roc_auc > .7 THEN 'not great' + ELSE 'poor' END AS model_quality, + accuracy, + precision, + recall FROM - ML.EVALUATE(MODEL ecommerce.classification_model_2, ( + ML.EVALUATE(MODEL ecommerce.classification_model_2, ( -WITH all_visitor_stats AS ( -SELECT - fullvisitorid, - IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit - FROM `data-to-insights.ecommerce.web_analytics` - GROUP BY fullvisitorid -) + WITH all_visitor_stats AS ( + SELECT + fullvisitorid, + IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit + FROM `data-to-insights.ecommerce.web_analytics` + GROUP BY fullvisitorid + ) # add in new features SELECT * EXCEPT(unique_session_id) FROM ( - SELECT - CONCAT(fullvisitorid, CAST(visitId AS STRING)) AS unique_session_id, - - # labels - will_buy_on_return_visit, - - MAX(CAST(h.eCommerceAction.action_type AS INT64)) AS latest_ecommerce_progress, - - # behavior on the site - IFNULL(totals.bounces, 0) AS bounces, - IFNULL(totals.timeOnSite, 0) AS time_on_site, - totals.pageviews, - - # where the visitor came from - trafficSource.source, - trafficSource.medium, - channelGrouping, - - # mobile or desktop - device.deviceCategory, - - # geographic - IFNULL(geoNetwork.country, "") AS country - - FROM `data-to-insights.ecommerce.web_analytics`, - UNNEST(hits) AS h - - JOIN all_visitor_stats USING(fullvisitorid) - - WHERE 1=1 - # only predict for new visits - AND totals.newVisits = 1 - AND date BETWEEN '20170501' AND '20170630' # eval 2 months - - GROUP BY - unique_session_id, - will_buy_on_return_visit, - bounces, - time_on_site, - totals.pageviews, - trafficSource.source, - trafficSource.medium, - channelGrouping, - device.deviceCategory, - country -) -)); + SELECT + CONCAT(fullvisitorid, CAST(visitId AS STRING)) AS unique_session_id, + + # labels + will_buy_on_return_visit, + + MAX(CAST(h.eCommerceAction.action_type AS INT64)) AS latest_ecommerce_progress, + + # behavior on the site + IFNULL(totals.bounces, 0) AS bounces, + IFNULL(totals.timeOnSite, 0) AS time_on_site, + totals.pageviews, + + # where the visitor came from + trafficSource.source, + trafficSource.medium, + channelGrouping, + + # mobile or desktop + device.deviceCategory, + + # geographic + IFNULL(geoNetwork.country, "") AS country + + FROM `data-to-insights.ecommerce.web_analytics`, + UNNEST(hits) AS h + + JOIN + all_visitor_stats + + USING(fullvisitorid) + + WHERE 1=1 + # only predict for new visits + AND totals.newVisits = 1 + AND date BETWEEN '20170501' AND '20170630' # eval 2 months + + GROUP BY + unique_session_id, + will_buy_on_return_visit, + bounces, + time_on_site, + totals.pageviews, + trafficSource.source, + trafficSource.medium, + channelGrouping, + device.deviceCategory, + country + ) + ) +); diff --git a/analytics/bigquery_ml_classification_logistic_regression_predict_with_model2.sql b/analytics/bigquery_ml_classification_logistic_regression_predict_with_model2.sql index f948030..fa36f31 100644 --- a/analytics/bigquery_ml_classification_logistic_regression_predict_with_model2.sql +++ b/analytics/bigquery_ml_classification_logistic_regression_predict_with_model2.sql @@ -2,86 +2,86 @@ -- Author: Hari Sekhon -- Date: 2020-09-06 10:05:48 +0100 (Sun, 06 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- use the improved model 2 to predict which users will buy on return visit SELECT - * + * FROM - ml.PREDICT( - MODEL `ecommerce.classification_model_2`, ( + ml.PREDICT( + MODEL `ecommerce.classification_model_2`, ( - WITH all_visitor_stats AS ( - SELECT - fullvisitorid, - IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit - FROM `data-to-insights.ecommerce.web_analytics` - GROUP BY fullvisitorid - ) + WITH all_visitor_stats AS ( + SELECT + fullvisitorid, + IF(COUNTIF(totals.transactions > 0 AND totals.newVisits IS NULL) > 0, 1, 0) AS will_buy_on_return_visit + FROM `data-to-insights.ecommerce.web_analytics` + GROUP BY fullvisitorid + ) - SELECT - CONCAT(fullvisitorid, '-',CAST(visitId AS STRING)) AS unique_session_id, + SELECT + CONCAT(fullvisitorid, '-',CAST(visitId AS STRING)) AS unique_session_id, - -- labels - will_buy_on_return_visit, + -- labels + will_buy_on_return_visit, - MAX(CAST(h.eCommerceAction.action_type AS INT64)) AS latest_ecommerce_progress, + MAX(CAST(h.eCommerceAction.action_type AS INT64)) AS latest_ecommerce_progress, - -- behavior on the site - IFNULL(totals.bounces, 0) AS bounces, - IFNULL(totals.timeOnSite, 0) AS time_on_site, - totals.pageviews, + -- behavior on the site + IFNULL(totals.bounces, 0) AS bounces, + IFNULL(totals.timeOnSite, 0) AS time_on_site, + totals.pageviews, - -- where the visitor came from - trafficSource.source, - trafficSource.medium, - channelGrouping, + -- where the visitor came from + trafficSource.source, + trafficSource.medium, + channelGrouping, - -- mobile or desktop - device.deviceCategory, + -- mobile or desktop + device.deviceCategory, - -- geographic - IFNULL(geoNetwork.country, "") AS country + -- geographic + IFNULL(geoNetwork.country, "") AS country - FROM - `data-to-insights.ecommerce.web_analytics`, - UNNEST(hits) AS h + FROM + `data-to-insights.ecommerce.web_analytics`, + UNNEST(hits) AS h - JOIN - all_visitor_stats - USING - (fullvisitorid) + JOIN + all_visitor_stats + USING + (fullvisitorid) - WHERE - -- only predict for new visits - totals.newVisits = 1 - -- test on a non-overlapping subset to the training data - AND date BETWEEN '20170701' AND '20170801' + WHERE + -- only predict for new visits + totals.newVisits = 1 + -- test on a non-overlapping subset to the training data + AND date BETWEEN '20170701' AND '20170801' - GROUP BY - unique_session_id, - will_buy_on_return_visit, - bounces, - time_on_site, - totals.pageviews, - trafficSource.source, - trafficSource.medium, - channelGrouping, - device.deviceCategory, - country + GROUP BY + unique_session_id, + will_buy_on_return_visit, + bounces, + time_on_site, + totals.pageviews, + trafficSource.source, + trafficSource.medium, + channelGrouping, + device.deviceCategory, + country + ) ) - ) ORDER BY - -- your label name will be prefixed with 'predicted_' for the field generated by the model - predicted_will_buy_on_return_visit DESC; + -- your label name will be prefixed with 'predicted_' for the field generated by the model + predicted_will_buy_on_return_visit DESC; diff --git a/aws_athena_cloudtrail_ddl.sql b/aws_athena_cloudtrail_ddl.sql index 9e0058a..11f9c86 100644 --- a/aws_athena_cloudtrail_ddl.sql +++ b/aws_athena_cloudtrail_ddl.sql @@ -4,14 +4,14 @@ -- -- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn -- and optionally send me feedback to help improve or steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- replace and on last line diff --git a/bigquery_billing_commonly_charged_units_of_measure.sql b/bigquery_billing_commonly_charged_units_of_measure.sql index bc2ff2b..e62eda2 100644 --- a/bigquery_billing_commonly_charged_units_of_measure.sql +++ b/bigquery_billing_commonly_charged_units_of_measure.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-02-02 00:37:06 +0000 (Sun, 02 Feb 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- BigQuery Billing Query @@ -18,13 +18,13 @@ -- commonly charged units of measure SELECT - usage_unit, - COUNT(*) AS billing_records + usage_unit, + COUNT(*) AS billing_records FROM - `myproject.mydata.imported_billing_data` + `myproject.mydata.imported_billing_data` WHERE - cost > 0 + cost > 0 GROUP BY - usage_unit + usage_unit ORDER BY - billing_records DESC; + billing_records DESC; diff --git a/bigquery_billing_latest_100_charges.sql b/bigquery_billing_latest_100_charges.sql index c9fe119..e438950 100644 --- a/bigquery_billing_latest_100_charges.sql +++ b/bigquery_billing_latest_100_charges.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-02-02 00:37:06 +0000 (Sun, 02 Feb 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- BigQuery Billing Query @@ -18,22 +18,22 @@ -- latest 100 charges SELECT - product, - resource_type, - start_time, - end_time, - cost, - project_id, - project_name, - project_labels_key, - currency, - currency_conversion_rate, - usage_amount, - usage_unit + product, + resource_type, + start_time, + end_time, + cost, + project_id, + project_name, + project_labels_key, + currency, + currency_conversion_rate, + usage_amount, + usage_unit FROM - `myproject.mydataset.imported_billing_data` + `myproject.mydataset.imported_billing_data` WHERE - cost > 0 + cost > 0 ORDER BY - end_time DESC + end_time DESC LIMIT 100; diff --git a/bigquery_billing_products_most_frequently_used.sql b/bigquery_billing_products_most_frequently_used.sql index 56428a3..f564208 100644 --- a/bigquery_billing_products_most_frequently_used.sql +++ b/bigquery_billing_products_most_frequently_used.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-02-02 00:37:06 +0000 (Sun, 02 Feb 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- BigQuery Billing Query @@ -18,13 +18,13 @@ -- most frequently used products costing more than 1 dollar SELECT - product, - COUNT(*) AS billing_records + product, + COUNT(*) AS billing_records FROM - `myproject.mydataset.imported_billing_data` + `myproject.mydataset.imported_billing_data` WHERE - cost > 1 + cost > 1 GROUP BY - product + product ORDER BY - billing_records DESC; + billing_records DESC; diff --git a/bigquery_billing_products_with_highest_aggregate_cost.sql b/bigquery_billing_products_with_highest_aggregate_cost.sql index efa718c..62a5097 100644 --- a/bigquery_billing_products_with_highest_aggregate_cost.sql +++ b/bigquery_billing_products_with_highest_aggregate_cost.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-02-02 00:37:06 +0000 (Sun, 02 Feb 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- BigQuery Billing Query @@ -18,11 +18,11 @@ -- products with highest aggregate cost SELECT - product, - ROUND(SUM(cost), 2) AS total_cost + product, + ROUND(SUM(cost), 2) AS total_cost FROM - `myproject.mydataset.imported_billing_data` + `myproject.mydataset.imported_billing_data` GROUP BY - product + product ORDER BY - total_cost DESC; + total_cost DESC; diff --git a/bigquery_billing_products_with_most_billing_records.sql b/bigquery_billing_products_with_most_billing_records.sql index dcf0f9a..7a66074 100644 --- a/bigquery_billing_products_with_most_billing_records.sql +++ b/bigquery_billing_products_with_most_billing_records.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-02-02 00:37:06 +0000 (Sun, 02 Feb 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- BigQuery Billing Query @@ -18,11 +18,11 @@ -- products with most billing records SELECT - product, - COUNT(*) AS billing_records + product, + COUNT(*) AS billing_records FROM - `myproject.mydataset.imported_billing_data` + `myproject.mydataset.imported_billing_data` GROUP BY - product + product ORDER BY - billing_records DESC; + billing_records DESC; diff --git a/bigquery_info_biggest_public_tables_by_row_count.sql b/bigquery_info_biggest_public_tables_by_row_count.sql index cb517a4..2f04fe8 100644 --- a/bigquery_info_biggest_public_tables_by_row_count.sql +++ b/bigquery_info_biggest_public_tables_by_row_count.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2020-09-16 08:48:42 +0100 (Wed, 16 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- @@ -24,7 +24,7 @@ -- -- in the DevOps Bash tools repo: -- --- https://github.com/HariSekhon/devops-bash-tools +-- https://github.com/HariSekhon/DevOps-Bash-tools -- Unfortunately we can't do this trick for the bigquery-public-data project because of this error: -- @@ -33,45 +33,45 @@ -- BigQuery error in ls operation: Access Denied: Project bigquery-public-data: User does not have bigquery.jobs.list permission in project bigquery-public-data. WITH ALL__TABLES__ AS ( - SELECT * FROM `bigquery-public-data.baseball.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.bls.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.census_bureau_usa.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.cloud_storage_geo_index.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.cms_codes.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.fec.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.genomics_cannabis.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.ghcn_d.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.ghcn_m.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.github_repos.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.hacker_news.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.irs_990.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.medicare.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.new_york.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.nlm_rxnorm.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.noaa_gsod.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.open_images.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.samples.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.san_francisco.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.stackoverflow.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.usa_names.__TABLES__` UNION ALL - SELECT * FROM `bigquery-public-data.utility_us.__TABLES__` + SELECT * FROM `bigquery-public-data.baseball.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.bls.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.census_bureau_usa.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.cloud_storage_geo_index.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.cms_codes.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.fec.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.genomics_cannabis.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.ghcn_d.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.ghcn_m.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.github_repos.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.hacker_news.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.irs_990.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.medicare.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.new_york.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.nlm_rxnorm.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.noaa_gsod.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.open_images.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.samples.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.san_francisco.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.stackoverflow.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.usa_names.__TABLES__` UNION ALL + SELECT * FROM `bigquery-public-data.utility_us.__TABLES__` ) SELECT - project_id, - dataset_id, - table_id, - row_count, - ROUND(size_bytes/pow(10,9),2) as size_gb, - TIMESTAMP_MILLIS(creation_time) AS creation_time, - TIMESTAMP_MILLIS(last_modified_time) as last_modified_time, - CASE - WHEN type = 1 THEN 'table' - WHEN type = 2 THEN 'view' - ELSE NULL - END AS type + project_id, + dataset_id, + table_id, + row_count, + ROUND(size_bytes/pow(10,9),2) as size_gb, + TIMESTAMP_MILLIS(creation_time) AS creation_time, + TIMESTAMP_MILLIS(last_modified_time) as last_modified_time, + CASE + WHEN type = 1 THEN 'table' + WHEN type = 2 THEN 'view' + ELSE NULL + END AS type FROM - ALL__TABLES__ + ALL__TABLES__ ORDER BY - row_count DESC, - size_gb DESC + row_count DESC, + size_gb DESC LIMIT 10; diff --git a/bigquery_info_columns.sql b/bigquery_info_columns.sql index 753064e..cdf280c 100644 --- a/bigquery_info_columns.sql +++ b/bigquery_info_columns.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2020-09-16 08:41:30 +0100 (Wed, 16 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- @@ -22,21 +22,21 @@ -- https://cloud.google.com/bigquery/docs/tables#columns_view SELECT - table_catalog, - table_schema, - table_name, - column_name, - ordinal_position, - is_nullable, - data_type, - is_generated, - generation_expression, - is_stored, - is_hidden, - is_updatable, - is_system_defined, - is_partitioning_column, - clustering_ordinal_position + table_catalog, + table_schema, + table_name, + column_name, + ordinal_position, + is_nullable, + data_type, + is_generated, + generation_expression, + is_stored, + is_hidden, + is_updatable, + is_system_defined, + is_partitioning_column, + clustering_ordinal_position FROM - -- XXX: replace bigquery-public-data.github_repos with myproject.mydataset - `bigquery-public-data.github_repos.INFORMATION_SCHEMA.COLUMNS`; + -- XXX: replace bigquery-public-data.github_repos with myproject.mydataset + `bigquery-public-data.github_repos.INFORMATION_SCHEMA.COLUMNS`; diff --git a/bigquery_info_columns_partitioned_clustered.sql b/bigquery_info_columns_partitioned_clustered.sql index fd1921b..8350186 100644 --- a/bigquery_info_columns_partitioned_clustered.sql +++ b/bigquery_info_columns_partitioned_clustered.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2020-09-16 08:44:33 +0100 (Wed, 16 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- @@ -21,11 +21,11 @@ SELECT - * + * FROM - -- XXX: replace bigquery-public-data.github_repos with myproject.mydataset - `bigquery-public-data.github_repos.INFORMATION_SCHEMA.COLUMNS` + -- XXX: replace bigquery-public-data.github_repos with myproject.mydataset + `bigquery-public-data.github_repos.INFORMATION_SCHEMA.COLUMNS` WHERE - is_partitioning_column = 'YES' - OR - clustering_ordinal_position IS NOT NULL; + is_partitioning_column = 'YES' + OR + clustering_ordinal_position IS NOT NULL; diff --git a/bigquery_info_datasets.sql b/bigquery_info_datasets.sql index e5709ad..0b0d5f5 100644 --- a/bigquery_info_datasets.sql +++ b/bigquery_info_datasets.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2020-09-16 08:29:55 +0100 (Wed, 16 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- @@ -20,17 +20,17 @@ -- Processes 20MB SELECT - s.*, - TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), creation_time, DAY) AS days_live, - option_value AS dataset_description + s.*, + TIMESTAMP_DIFF(CURRENT_TIMESTAMP(), creation_time, DAY) AS days_live, + option_value AS dataset_description FROM - -- XXX: can prefix INFORMATION_SCHEMA with your '.' to select from another project, but defaulting to the current project is more convenient than editing this - `INFORMATION_SCHEMA.SCHEMATA` AS s - LEFT JOIN `INFORMATION_SCHEMA.SCHEMATA_OPTIONS` AS so - USING (schema_name) + -- XXX: can prefix INFORMATION_SCHEMA with your '.' to select from another project, but defaulting to the current project is more convenient than editing this + `INFORMATION_SCHEMA.SCHEMATA` AS s + LEFT JOIN `INFORMATION_SCHEMA.SCHEMATA_OPTIONS` AS so + USING (schema_name) -- will miss all the datasets without descriptions like this --WHERE --- so.option_name = 'description' +-- so.option_name = 'description' ORDER BY - last_modified_time DESC + last_modified_time DESC LIMIT 15; diff --git a/bigquery_info_tables.sql b/bigquery_info_tables.sql index c2ca58c..dbec288 100644 --- a/bigquery_info_tables.sql +++ b/bigquery_info_tables.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2020-09-16 08:29:55 +0100 (Wed, 16 Sep 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- @@ -37,21 +37,21 @@ -- SELECT - dataset_id, - table_id, - -- convert bytes to GB - ROUND(size_bytes/pow(10,9),2) as size_gb, - row_count, - -- Convert UNIX EPOCH to a timestamp. - TIMESTAMP_MILLIS(creation_time) AS creation_time, - TIMESTAMP_MILLIS(last_modified_time) as last_modified_time, - CASE - WHEN type = 1 THEN 'table' - WHEN type = 2 THEN 'view' - ELSE NULL - END AS type + dataset_id, + table_id, + -- convert bytes to GB + ROUND(size_bytes/pow(10,9),2) as size_gb, + row_count, + -- Convert UNIX EPOCH to a timestamp. + TIMESTAMP_MILLIS(creation_time) AS creation_time, + TIMESTAMP_MILLIS(last_modified_time) as last_modified_time, + CASE + WHEN type = 1 THEN 'table' + WHEN type = 2 THEN 'view' + ELSE NULL + END AS type FROM - -- XXX: replace bigquery-public-data.github_repos with myproject.mydataset - `bigquery-public-data.github_repos.__TABLES__` + -- XXX: replace bigquery-public-data.github_repos with myproject.mydataset + `bigquery-public-data.github_repos.__TABLES__` ORDER BY - size_gb DESC; + size_gb DESC; diff --git a/mysql_databases_by_size.sql b/mysql_databases_by_size.sql index 94d298c..84d090c 100644 --- a/mysql_databases_by_size.sql +++ b/mysql_databases_by_size.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-07 01:00:25 +0100 (Fri, 07 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL databases by size descending @@ -18,12 +18,12 @@ -- Tested on MySQL 5.5, 5.6, 5.7, 8.0 and MariaDB 5.5, 10.0 - 10.5 SELECT - table_schema, - ROUND(SUM(data_length + index_length) / 1024 / 1024, 3) AS 'Database Size (MB)', - ROUND(SUM(data_free) / 1024 / 1024, 3) AS 'Free Space (MB)' + table_schema, + ROUND(SUM(data_length + index_length) / 1024 / 1024, 3) AS 'Database Size (MB)', + ROUND(SUM(data_free) / 1024 / 1024, 3) AS 'Free Space (MB)' FROM - information_schema.tables + information_schema.tables GROUP BY - table_schema + table_schema ORDER BY - 2 DESC; + 2 DESC; diff --git a/mysql_host_summary.sql b/mysql_host_summary.sql index 9f76a04..1ef1b7f 100644 --- a/mysql_host_summary.sql +++ b/mysql_host_summary.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 01:51:14 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL show activity per host @@ -23,8 +23,8 @@ -- sys doesn't exist in MySQL <= 5.6 or MariaDB 10.5 SELECT - * + * FROM - sys.host_summary + sys.host_summary ORDER BY - statement_latency DESC; + statement_latency DESC; diff --git a/mysql_indexes_unused.sql b/mysql_indexes_unused.sql index 1228070..5e509c5 100644 --- a/mysql_indexes_unused.sql +++ b/mysql_indexes_unused.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 02:41:02 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL unused indexes @@ -35,10 +35,10 @@ SELECT IF (@@performance_schema, 'TRUE', 'FALSE') AS 'Performance Schema enabled \! echo "Unused Indexes since startup:"; SELECT - * + * FROM - sys.schema_unused_indexes + sys.schema_unused_indexes ORDER BY - object_schema, - object_name, - index_name; + object_schema, + object_name, + index_name; diff --git a/mysql_info.sql b/mysql_info.sql index 6d23485..289d7cf 100644 --- a/mysql_info.sql +++ b/mysql_info.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 10:08:00 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Useful MySQL funcs & info @@ -25,18 +25,18 @@ -- some of these allow not using the brackets but use them for portability between MySQL 5.x and 8.x SELECT - VERSION(), - USER(), -- client's reported user + host, same as SESSION_USER(), SYSTEM_USER() - CURRENT_USER(), -- authenticated user name + host name - this is the one you want for debugging your mysql.user table configuration - DATABASE(), -- SCHEMA() - NOW(), - CURDATE(), - CURTIME(), - UTC_DATE(), - UTC_TIME(), - UTC_TIMESTAMP(), - SYSDATE(), -- returns date of function completion - -- PS_CURRENT_THREAD_ID(), -- MySQL 8.0.16+ - UUID_SHORT(), -- integer - UUID() -- alnum + VERSION(), + USER(), -- client's reported user + host, same as SESSION_USER(), SYSTEM_USER() + CURRENT_USER(), -- authenticated user name + host name - this is the one you want for debugging your mysql.user table configuration + DATABASE(), -- SCHEMA() + NOW(), + CURDATE(), + CURTIME(), + UTC_DATE(), + UTC_TIME(), + UTC_TIMESTAMP(), + SYSDATE(), -- returns date of function completion + -- PS_CURRENT_THREAD_ID(), -- MySQL 8.0.16+ + UUID_SHORT(), -- integer + UUID() -- alnum ; diff --git a/mysql_memory_by_host.sql b/mysql_memory_by_host.sql index ccd0029..0f46814 100644 --- a/mysql_memory_by_host.sql +++ b/mysql_memory_by_host.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 02:13:54 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL memory used per host @@ -23,10 +23,10 @@ -- sys doesn't exist in MySQL <= 5.6 or MariaDB 10.5 SELECT - * + * FROM - sys.memory_by_host_by_current_bytes + sys.memory_by_host_by_current_bytes WHERE - host <> 'background' + host <> 'background' ORDER BY - total_allocated DESC; + total_allocated DESC; diff --git a/mysql_memory_by_user.sql b/mysql_memory_by_user.sql index 0338f39..998bc60 100644 --- a/mysql_memory_by_user.sql +++ b/mysql_memory_by_user.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 02:13:54 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL memory used per user @@ -23,10 +23,10 @@ -- sys doesn't exist in MySQL <= 5.6 or MariaDB 10.5 SELECT - * + * FROM - sys.memory_by_user_by_current_bytes + sys.memory_by_user_by_current_bytes WHERE - user <> 'background' + user <> 'background' ORDER BY - total_allocated DESC; + total_allocated DESC; diff --git a/mysql_sessions.sql b/mysql_sessions.sql index 92bc42a..64cfaeb 100644 --- a/mysql_sessions.sql +++ b/mysql_sessions.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 02:13:54 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL sessions @@ -23,31 +23,31 @@ -- sys doesn't exist in MySQL <= 5.6 or MariaDB 10.5 SELECT - user, - db, - program_name, - command, - state, - time, - current_statement, - current_memory, - progress, - lock_latency, - rows_examined, - rows_sent, - rows_affected, - tmp_tables, - tmp_disk_tables, - full_scan, - last_statement, - last_statement_latency, - last_wait, - last_wait_latency, - trx_latency, - trx_state + user, + db, + program_name, + command, + state, + time, + current_statement, + current_memory, + progress, + lock_latency, + rows_examined, + rows_sent, + rows_affected, + tmp_tables, + tmp_disk_tables, + full_scan, + last_statement, + last_statement_latency, + last_wait, + last_wait_latency, + trx_latency, + trx_state FROM - sys.session + sys.session WHERE - user <> 'sql/event_scheduler' + user <> 'sql/event_scheduler' ORDER BY - current_memory DESC; + current_memory DESC; diff --git a/mysql_statement_latency_by_host.sql b/mysql_statement_latency_by_host.sql index db94dbf..38e2d72 100644 --- a/mysql_statement_latency_by_host.sql +++ b/mysql_statement_latency_by_host.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 01:51:14 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL statement latency stats per host @@ -23,8 +23,8 @@ -- sys doesn't exist in MySQL <= 5.6 or MariaDB 10.5 SELECT - * + * FROM - sys.host_summary_by_statement_latency + sys.host_summary_by_statement_latency ORDER BY - total_latency DESC; + total_latency DESC; diff --git a/mysql_statement_latency_by_user.sql b/mysql_statement_latency_by_user.sql index 1315b6b..202d68d 100644 --- a/mysql_statement_latency_by_user.sql +++ b/mysql_statement_latency_by_user.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 01:51:14 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL statement latency stats per user @@ -23,8 +23,8 @@ -- sys doesn't exist in MySQL <= 5.6 or MariaDB 10.5 SELECT - * + * FROM - sys.user_summary_by_statement_latency + sys.user_summary_by_statement_latency ORDER BY - total_latency DESC; + total_latency DESC; diff --git a/mysql_tables.sql b/mysql_tables.sql index ef340af..09cb209 100644 --- a/mysql_tables.sql +++ b/mysql_tables.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 02:45:44 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL tables across schemas (useful for finding tables) @@ -18,24 +18,24 @@ -- Tested on MySQL 5.5, 5.6, 5.7, 8.0 and MariaDB 5.5, 10.0 - 10.5 SELECT - table_schema, - table_name, - table_type, - engine, - table_rows, - avg_row_length, - data_length, - max_data_length, - index_length, - data_free, - auto_increment, - create_time, - update_time, - table_comment + table_schema, + table_name, + table_type, + engine, + table_rows, + avg_row_length, + data_length, + max_data_length, + index_length, + data_free, + auto_increment, + create_time, + update_time, + table_comment FROM - information_schema.tables + information_schema.tables WHERE - table_type NOT LIKE '%VIEW%' + table_type NOT LIKE '%VIEW%' ORDER BY - table_schema, - table_name; + table_schema, + table_name; diff --git a/mysql_tables_nonsystem.sql b/mysql_tables_nonsystem.sql index a4b74d0..217db7d 100644 --- a/mysql_tables_nonsystem.sql +++ b/mysql_tables_nonsystem.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 02:45:44 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL tables across schemas, excluding system tables such as information_schema, performance_schema, mysql and sys @@ -18,26 +18,26 @@ -- Tested on MySQL 5.5, 5.6, 5.7, 8.0 and MariaDB 5.5, 10.0 - 10.5 SELECT - table_schema, - table_name, - table_type, - engine, - table_rows, - avg_row_length, - data_length, - max_data_length, - index_length, - data_free, - auto_increment, - create_time, - update_time, - table_comment + table_schema, + table_name, + table_type, + engine, + table_rows, + avg_row_length, + data_length, + max_data_length, + index_length, + data_free, + auto_increment, + create_time, + update_time, + table_comment FROM - information_schema.tables + information_schema.tables WHERE - table_type NOT LIKE '%VIEW%' - AND - table_schema NOT IN ('mysql', 'sys', 'information_schema', 'performance_schema') + table_type NOT LIKE '%VIEW%' + AND + table_schema NOT IN ('mysql', 'sys', 'information_schema', 'performance_schema') ORDER BY - table_schema, - table_name; + table_schema, + table_name; diff --git a/mysql_user.sql b/mysql_user.sql index d9bef20..2cda08f 100644 --- a/mysql_user.sql +++ b/mysql_user.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 10:08:00 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL Requested vs Actual User @@ -22,6 +22,6 @@ -- https://dev.mysql.com/doc/refman/8.0/en/sql-function-reference.html SELECT - USER() AS 'Your Requested USER()', -- client's reported user + host, same as SESSION_USER(), SYSTEM_USER() - CURRENT_USER() AS 'Your Actual CURRENT_USER()' -- authenticated user name + host name - this is the one you want for debugging your mysql.user table configuration + USER() AS 'Your Requested USER()', -- client's reported user + host, same as SESSION_USER(), SYSTEM_USER() + CURRENT_USER() AS 'Your Actual CURRENT_USER()' -- authenticated user name + host name - this is the one you want for debugging your mysql.user table configuration ; diff --git a/mysql_user_summary.sql b/mysql_user_summary.sql index ec39724..f2f197e 100644 --- a/mysql_user_summary.sql +++ b/mysql_user_summary.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 01:51:14 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL show activity per user @@ -23,8 +23,8 @@ -- sys doesn't exist in MySQL <= 5.6 or MariaDB 10.5 SELECT - * + * FROM - sys.user_summary + sys.user_summary ORDER BY - statement_latency DESC; + statement_latency DESC; diff --git a/mysql_users.sql b/mysql_users.sql index 78a227a..078f206 100644 --- a/mysql_users.sql +++ b/mysql_users.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 09:51:00 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List MySQL users ordered by host, user similar to auth resolution @@ -23,18 +23,18 @@ -- some fields don't exist on MySQL < 8.0 and MariaDB 10.0 - 10.5, must use mysql_users_pre8.sql instead SELECT - host, - user, - max_connections, - max_user_connections, - password_expired, - password_last_changed, - password_lifetime, - account_locked, - password_reuse_time, - password_require_current + host, + user, + max_connections, + max_user_connections, + password_expired, + password_last_changed, + password_lifetime, + account_locked, + password_reuse_time, + password_require_current FROM - mysql.user + mysql.user ORDER BY - host, - user; + host, + user; diff --git a/mysql_users_pre56.sql b/mysql_users_pre56.sql index 5652aad..937b085 100644 --- a/mysql_users_pre56.sql +++ b/mysql_users_pre56.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 09:51:00 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List MySQL users ordered by host, user similar to auth resolution @@ -18,20 +18,20 @@ -- Tested on MySQL 5.5, 5.6, 5.7, 8.0 and MariaDB 5.5, 10.0 - 10.5 SELECT - host, - user, - max_connections, - max_user_connections - -- this field doesn't exist in MySQL < 5.6 - -- password_expired - -- these fields don't exist in MySQL < 8.0 / MariaDB 10.5 - -- password_last_changed, - -- password_lifetime, - -- account_locked, - -- password_reuse_time, - -- password_require_current + host, + user, + max_connections, + max_user_connections + -- this field doesn't exist in MySQL < 5.6 + -- password_expired + -- these fields don't exist in MySQL < 8.0 / MariaDB 10.5 + -- password_last_changed, + -- password_lifetime, + -- account_locked, + -- password_reuse_time, + -- password_require_current FROM - mysql.user + mysql.user ORDER BY - host, - user; + host, + user; diff --git a/mysql_users_pre8.sql b/mysql_users_pre8.sql index b9bcb5c..aad0005 100644 --- a/mysql_users_pre8.sql +++ b/mysql_users_pre8.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 09:51:00 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List MySQL users ordered by host, user similar to auth resolution @@ -21,20 +21,20 @@ -- Tested on MySQL 5.6, 5.7, 8.0 and MariaDB 10.0 - 10.5 SELECT - host, - user, - max_connections, - max_user_connections, - -- not available on MySQL 5.5 - password_expired - -- these fields don't exist in MySQL < 8.0 / MariaDB 10 - -- password_last_changed, - -- password_lifetime, - -- account_locked, - -- password_reuse_time, - -- password_require_current + host, + user, + max_connections, + max_user_connections, + -- not available on MySQL 5.5 + password_expired + -- these fields don't exist in MySQL < 8.0 / MariaDB 10 + -- password_last_changed, + -- password_lifetime, + -- account_locked, + -- password_reuse_time, + -- password_require_current FROM - mysql.user + mysql.user ORDER BY - host, - user; + host, + user; diff --git a/mysql_views.sql b/mysql_views.sql index 85c3021..6c838ba 100644 --- a/mysql_views.sql +++ b/mysql_views.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-08 09:54:19 +0100 (Sat, 08 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL views across schemas (useful for finding views) @@ -18,14 +18,14 @@ -- Tested on MySQL 5.5, 5.6, 5.7, 8.0 and MariaDB 5.5, 10.0 - 10.5 SELECT - -- table_catalog, - table_schema, - table_name, - IS_UPDATABLE, - DEFINER, - SECURITY_TYPE + -- table_catalog, + table_schema, + table_name, + IS_UPDATABLE, + DEFINER, + SECURITY_TYPE FROM - information_schema.views + information_schema.views ORDER BY - table_schema, - table_name; + table_schema, + table_name; diff --git a/mysql_views_system.sql b/mysql_views_system.sql index cd2414b..18c2530 100644 --- a/mysql_views_system.sql +++ b/mysql_views_system.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 02:45:44 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- MySQL system views @@ -18,24 +18,24 @@ -- Tested on MySQL 5.5, 5.6, 5.7, 8.0 and MariaDB 5.5, 10.0 - 10.5 SELECT - table_schema, - table_name, - table_type, - engine, - table_rows, - avg_row_length, - data_length, - max_data_length, - index_length, - data_free, - auto_increment, - create_time, - update_time, - table_comment + table_schema, + table_name, + table_type, + engine, + table_rows, + avg_row_length, + data_length, + max_data_length, + index_length, + data_free, + auto_increment, + create_time, + update_time, + table_comment FROM - information_schema.tables + information_schema.tables WHERE - table_type = 'SYSTEM VIEW' + table_type = 'SYSTEM VIEW' ORDER BY - table_schema, - table_name; + table_schema, + table_name; diff --git a/oracle_checkpoints.sql b/oracle_checkpoints.sql new file mode 100644 index 0000000..19b7f5c --- /dev/null +++ b/oracle_checkpoints.sql @@ -0,0 +1,26 @@ +-- +-- Author: Hari Sekhon +-- Date: 2008-04-28 21:36:48 +0100 (Mon, 28 Apr 2008) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Checkpoints on DBF data files (relevant for DBA recoveries) +-- +-- Tested on Oracle 9i, 10g, 11g, 19c + +SELECT + NAME, + checkpoint_time +FROM + v$datafile +ORDER BY + checkpoint_time; diff --git a/oracle_datafiles_checkpoint_change.sql b/oracle_datafiles_checkpoint_change.sql new file mode 100644 index 0000000..a6f2efe --- /dev/null +++ b/oracle_datafiles_checkpoint_change.sql @@ -0,0 +1,26 @@ +-- +-- Author: Hari Sekhon +-- Date: 2008-04-28 21:36:48 +0100 (Mon, 28 Apr 2008) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle Show the Checkpoints on the DBF data files (important for DBA recoveries) +-- +-- Tested on Oracle 9i, 10g, 11g, 19c + +SELECT + checkpoint_change#, + file# +FROM + v$datafile +ORDER BY + checkpoint_change#; diff --git a/oracle_datafiles_checkpoint_time.sql b/oracle_datafiles_checkpoint_time.sql new file mode 100644 index 0000000..01992d8 --- /dev/null +++ b/oracle_datafiles_checkpoint_time.sql @@ -0,0 +1,27 @@ +-- +-- Author: Hari Sekhon +-- Date: 2008-04-28 21:36:48 +0100 (Mon, 28 Apr 2008) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle Show Checkpoint Dates on DBF data files (relevant for DBA recoveries) +-- +-- Tested on Oracle 9i, 10g, 11g, 19c + + +SELECT + checkpoint_time, + file# +FROM + v$datafile +ORDER BY + checkpoint_time; diff --git a/oracle_datafiles_lastchanged.sql b/oracle_datafiles_lastchanged.sql new file mode 100644 index 0000000..b2816a5 --- /dev/null +++ b/oracle_datafiles_lastchanged.sql @@ -0,0 +1,26 @@ +-- +-- Author: Hari Sekhon +-- Date: 2008-04-28 21:36:48 +0100 (Mon, 28 Apr 2008) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle Show the system change number (SCN) +-- +-- Tested on Oracle 9i, 10g, 11g, 19c + +SELECT + last_change#, + file# +FROM + v$datafile +ORDER BY + last_change#; diff --git a/oracle_datafiles_unrecoverable_change.sql b/oracle_datafiles_unrecoverable_change.sql new file mode 100644 index 0000000..9eeef9f --- /dev/null +++ b/oracle_datafiles_unrecoverable_change.sql @@ -0,0 +1,26 @@ +-- +-- Author: Hari Sekhon +-- Date: 2008-04-28 21:36:48 +0100 (Mon, 28 Apr 2008) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle +-- +-- Tested on Oracle 9i, 10g, 11g, 19c + +SELECT + unrecoverable_change#, + file# +FROM + v$datafile +ORDER BY + unrecoverable_change#; diff --git a/oracle_rds_ddl_tablespace.sql b/oracle_rds_ddl_tablespace.sql new file mode 100644 index 0000000..d6f9ffc --- /dev/null +++ b/oracle_rds_ddl_tablespace.sql @@ -0,0 +1,21 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-18 01:25:03 +0400 (Fri, 18 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - get the Tablespace DDL + +SELECT + -- USERS tablespace was case sensitive + dbms_metadata.get_ddl('TABLESPACE','USERS') +FROM dual; diff --git a/oracle_recent_queries.sql b/oracle_recent_queries.sql new file mode 100644 index 0000000..2331d3e --- /dev/null +++ b/oracle_recent_queries.sql @@ -0,0 +1,27 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-12 06:17:50 +0300 (Sat, 12 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Recent Queries +-- +-- Tested on Oracle 19c + +SELECT + sql_text, + executions +FROM + v$sql +ORDER BY + last_active_time +DESC; diff --git a/oracle_show_dba_recyclebin.sql b/oracle_show_dba_recyclebin.sql new file mode 100644 index 0000000..fca465d --- /dev/null +++ b/oracle_show_dba_recyclebin.sql @@ -0,0 +1,32 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-11 03:50:00 +0300 (Fri, 11 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show All Users Recyclebins via DBA Recyclebin +-- +-- Tested on Oracle 19c + +SELECT + owner, + object_name, + original_name, + type, + droptime, + space +FROM + dba_recyclebin +ORDER BY + owner, + droptime +DESC; diff --git a/oracle_show_sessions_using_temp_tablespace.sql b/oracle_show_sessions_using_temp_tablespace.sql new file mode 100644 index 0000000..b0908ea --- /dev/null +++ b/oracle_show_sessions_using_temp_tablespace.sql @@ -0,0 +1,35 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-17 23:06:57 +0400 (Thu, 17 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show User Sessions Using Temporary Tablespace +-- +-- Check there are none before dropping an old temp tablespace file to avoid disruptions +-- +-- Tested on Oracle 19c + +SELECT + s.sid, + s.username, + t.tablespace, + t.blocks, + t.segfile#, + t.segblk#, + t.contents, + t.sql_id +FROM + v$sort_usage t, + v$session s +WHERE + t.session_addr = s.saddr; diff --git a/oracle_table_segments_for_schema.sql b/oracle_table_segments_for_schema.sql new file mode 100644 index 0000000..e52aecf --- /dev/null +++ b/oracle_table_segments_for_schema.sql @@ -0,0 +1,42 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-11 03:24:48 +0300 (Fri, 11 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Tables' Segments Size in a given Tablespace +-- +-- Tested on Oracle 19c + +SELECT + segment_name, + segment_type, + tablespace_name, + bytes/1024/1024/1024 AS size_gb, + blocks +FROM + dba_segments +WHERE + owner = 'USERS' -- XXX: Edit this + AND + segment_type = 'TABLE' + AND + blocks > 8 + -- to look at only specific tables + -- AND + --segment_name IN + --('MY_TABLE_1', + -- 'MY_TABLE_2', + -- 'MY_TABLE_3', + -- 'MY_TABLE_4') +ORDER BY + size_gb DESC; diff --git a/oracle_table_shrink_candidates.sql b/oracle_table_shrink_candidates.sql new file mode 100644 index 0000000..0ac0208 --- /dev/null +++ b/oracle_table_shrink_candidates.sql @@ -0,0 +1,36 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-11 03:24:48 +0300 (Fri, 11 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Table Candidates to Move / Shrink in a given Tablespace 'USERS' +-- +-- where the tables are over 20% utilized +-- +-- Tested on Oracle 19c + +SELECT + segment_name, + segment_type, + ROUND(SUM(bytes)/1024/1024/1024, 2) AS size_gb +FROM + dba_segments +WHERE + tablespace_name = 'USERS' -- XXX: Edit +GROUP BY + segment_name, + segment_type +HAVING + SUM(bytes)/1024/1024/1024 > 1 +ORDER BY + size_gb DESC; diff --git a/oracle_table_space.sql b/oracle_table_space.sql new file mode 100644 index 0000000..ad4cc53 --- /dev/null +++ b/oracle_table_space.sql @@ -0,0 +1,53 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-11 03:24:48 +0300 (Fri, 11 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Tables' Space Used vs Free and Free Percentage +-- +-- where the tables are over 20% utilized +-- +-- Calculations assume an 8KB block size, which you should verify like this: +-- +-- SELECT value FROM v$parameter WHERE name = 'db_block_size'; +-- +-- Tested on Oracle 19c + +SELECT + owner, + table_name, + -- each block is 8KB, multiply it to GB, round to two decimal places + ROUND(blocks * 8 / 1024 / 1024, 2) AS total_gb, + -- estimate data size from rows vs average row size, round to two decimal places + ROUND(num_rows * avg_row_len / 1024 / 1024 / 1024, 2) AS actual_data_gb, + -- estimate free space by subtracting the two above calculations + ROUND((blocks * 8 / 1024 / 1024) - (num_rows * avg_row_len / 1024 / 1024 / 1024), 2) AS free_space_gb, + -- calculate free space percentage from the above three calculations + ROUND( + ( (blocks * 8 / 1024 / 1024) - (num_rows * avg_row_len / 1024 / 1024 / 1024) ) / + (blocks * 8 / 1024 / 1024) * 100, 2) AS free_space_pct +FROM + dba_tables +WHERE + blocks > 0 + AND + num_rows > 0 + AND + ((blocks * 8 / 1024 / 1024) - (num_rows * avg_row_len / 1024 / 1024 / 1024)) / + (blocks * 8 / 1024 / 1024) > 0.2 -- TUNE: currently only showing tables over 20% utilized + AND + owner NOT IN + ('SYS', 'SYSTEM', 'SYSAUX', 'RDSADMIN') +ORDER BY + free_space_gb DESC, + total_gb DESC; diff --git a/oracle_table_space_for_schema.sql b/oracle_table_space_for_schema.sql new file mode 100644 index 0000000..4456738 --- /dev/null +++ b/oracle_table_space_for_schema.sql @@ -0,0 +1,61 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-11 03:24:48 +0300 (Fri, 11 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Tables' Space Used vs Free and Free Percentage in a given Tablespace +-- +-- for tables over 20% utilized +-- +-- Calculations assume an 8KB block size, which you should verify like this: +-- +-- SELECT value FROM v$parameter WHERE name = 'db_block_size'; +-- +-- Tested on Oracle 19c + +SELECT + t.owner, + t.table_name, + -- each block is 8KB, multiply it to GB, round to two decimal places + ROUND(t.blocks * 8 / 1024 / 1024, 2) AS total_gb_from_tables, + -- estimate from segments + ROUND(s.bytes / 1024 / 1024 / 1024, 2) AS total_gb_from_segments, + -- estimate data size from rows vs average row size, round to two decimal places + ROUND(t.num_rows * t.avg_row_len / 1024 / 1024 / 1024, 2) AS actual_data_gb, + -- estimate free space by subtracting the two above calculations + ROUND(((t.blocks * 8 / 1024) - (t.num_rows * t.avg_row_len / 1024 / 1024)) / 1024, 2) AS free_space_gb, + -- calculate free space percentage from the above three calculations + ROUND( + ( (t.blocks * 8) - (t.num_rows * t.avg_row_len / 1024) ) / + (t.blocks * 8) * 100, 2) AS free_space_pct, + t.last_analyzed +FROM + dba_tables t +JOIN + dba_segments s +ON + t.owner = s.owner + AND + t.table_name = s.segment_name +WHERE + t.blocks > 0 + AND + t.num_rows > 0 + AND + -- TUNE: currently only showing tables over 20% utilized + ((t.blocks * 8 / 1024) - (t.num_rows * t.avg_row_len / 1024 / 1024)) / (t.blocks * 8 / 1024) > 0.2 + AND + t.owner = 'USERS' -- XXX: Change this to your owner schema +ORDER BY + free_space_gb DESC, + total_gb_from_segments DESC; diff --git a/oracle_tablespace_datafiles_sum_vs_max_autoextend.sql b/oracle_tablespace_datafiles_sum_vs_max_autoextend.sql new file mode 100644 index 0000000..cde37c7 --- /dev/null +++ b/oracle_tablespace_datafiles_sum_vs_max_autoextend.sql @@ -0,0 +1,29 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-18 01:18:50 +0400 (Fri, 18 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Tablespace Datafiles Sum vs Max and AutoExtend + + +SELECT + tablespace_name, + ROUND(SUM(BYTES)/(1024*1024*1024),2) Total_GB, + ROUND(MAXBYTES/(1024*1024*1024),2) Max_GB, + autoextensible +FROM + dba_data_files +GROUP BY + tablespace_name, + maxbytes, + autoextensible; diff --git a/oracle_tablespace_datafiles_sum_vs_max_autoextend_temp.sql b/oracle_tablespace_datafiles_sum_vs_max_autoextend_temp.sql new file mode 100644 index 0000000..1aad957 --- /dev/null +++ b/oracle_tablespace_datafiles_sum_vs_max_autoextend_temp.sql @@ -0,0 +1,29 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-18 01:18:50 +0400 (Fri, 18 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Temp Tablespace Datafiles Sum vs Max and AutoExtend + + +SELECT + tablespace_name, + ROUND(SUM(BYTES)/(1024*1024*1024),2) Total_GB, + ROUND(MAXBYTES/(1024*1024*1024),2) Max_GB, + autoextensible +FROM + dba_temp_files +GROUP BY + tablespace_name, + maxbytes, + autoextensible; diff --git a/oracle_tablespace_space.sql b/oracle_tablespace_space.sql new file mode 100644 index 0000000..c9ddd35 --- /dev/null +++ b/oracle_tablespace_space.sql @@ -0,0 +1,42 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-11 03:24:48 +0300 (Fri, 11 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Tablespace Size, Space Used vs Free in GB and as a Percentage +-- +-- Tested on Oracle 19c + +SELECT + df.tablespace_name "Tablespace", + df.bytes / (1024 * 1024 * 1024) "Size (GB)", + (df.bytes - SUM(fs.bytes)) / (1024 * 1024 * 1024) "Used Space (GB)", + ROUND(SUM(fs.bytes) / (1024 * 1024 * 1024), 2) "Free Space (GB)", + ROUND(SUM(fs.bytes) / df.bytes * 100, 2) "Free Space %" +FROM + dba_free_space fs, + (SELECT + tablespace_name, + SUM(bytes) bytes + FROM + dba_data_files + GROUP BY + tablespace_name) df +WHERE + fs.tablespace_name (+) = df.tablespace_name +GROUP BY + df.tablespace_name, + df.bytes +ORDER BY + "Free Space (GB)" DESC, + "Used Space (GB)" DESC; diff --git a/oracle_tablespace_space2.sql b/oracle_tablespace_space2.sql new file mode 100644 index 0000000..7354551 --- /dev/null +++ b/oracle_tablespace_space2.sql @@ -0,0 +1,34 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-11 03:24:48 +0300 (Fri, 11 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Tablespace Size, Space Used GB and Percentage Used +-- +-- Calculations assume an 8KB block size, which you should verify like this: +-- +-- SELECT value FROM v$parameter WHERE name = 'db_block_size'; +-- +-- Tested on Oracle 19c + +SELECT + tablespace_name "Tablespace", + -- convert used_space in blocks to GB as each block is 8KB + ROUND(used_space * 8 / 1024 / 1024, 2) AS "Used Space (GB)", + -- convert tablespace_size in blocks to GB as each block is 8KB + ROUND(tablespace_size * 8 / 1024 / 1024, 2) AS "Total Space (GB)", + ROUND(used_percent, 2) AS "Used Space %" +FROM + dba_tablespace_usage_metrics +ORDER BY + "Used Space %" DESC; diff --git a/oracle_tablespace_undo_space.sql b/oracle_tablespace_undo_space.sql new file mode 100644 index 0000000..f172263 --- /dev/null +++ b/oracle_tablespace_undo_space.sql @@ -0,0 +1,44 @@ +-- +-- Author: Hari Sekhon +-- Date: 2024-10-18 04:24:34 +0400 (Fri, 18 Oct 2024) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https///github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show Tablespace size of the Undo Tablespace +-- +-- Tested on Oracle 19c + +SELECT + df.tablespace_name "Tablespace", + df.bytes / (1024 * 1024 * 1024) "Size (GB)", + (df.bytes - SUM(fs.bytes)) / (1024 * 1024 * 1024) "Used Space (GB)", + ROUND(SUM(fs.bytes) / (1024 * 1024 * 1024), 2) "Free Space (GB)", + ROUND(SUM(fs.bytes) / df.bytes * 100, 2) "Free Space %" +FROM + dba_free_space fs, + (SELECT + tablespace_name, + SUM(bytes) bytes + FROM + dba_data_files + GROUP BY + tablespace_name) df +WHERE + fs.tablespace_name (+) = df.tablespace_name + AND + UPPER(fs.tablespace_name) LIKE '%UNDO%' +GROUP BY + df.tablespace_name, + df.bytes +ORDER BY + "Free Space (GB)" DESC, + "Used Space (GB)" DESC; diff --git a/oracle_user_sessions.sql b/oracle_user_sessions.sql new file mode 100644 index 0000000..7f309e5 --- /dev/null +++ b/oracle_user_sessions.sql @@ -0,0 +1,26 @@ +-- +-- Author: Hari Sekhon +-- Date: 2008-04-28 21:36:48 +0100 (Mon, 28 Apr 2008) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- Oracle - Show User Sessions Status +-- +-- Tested on Oracle 9i, 10g, 11g, 19c + +SELECT + username, + sid, + serial#, + status +FROM + v$session; diff --git a/postgres_active_query_count.sql b/postgres_active_query_count.sql index 8ef918e..1f2de24 100644 --- a/postgres_active_query_count.sql +++ b/postgres_active_query_count.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 15:33:36 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL number of active queries @@ -22,8 +22,8 @@ -- Tested on PostgreSQL 9.2+, 10.x, 11.x, 12.x, 13.0 SELECT - COUNT(*) as active_query_count + COUNT(*) as active_query_count FROM - pg_stat_activity + pg_stat_activity WHERE - state='active'; + state='active'; diff --git a/postgres_backends_per_database.sql b/postgres_backends_per_database.sql index 8d36767..9e61e64 100644 --- a/postgres_backends_per_database.sql +++ b/postgres_backends_per_database.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-17 20:17:18 +0100 (Mon, 17 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL number of backends connected to each database @@ -18,10 +18,10 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - datname, - numbackends + datname, + numbackends FROM - pg_catalog.pg_stat_database + pg_catalog.pg_stat_database ORDER BY - numbackends DESC, - datname ASC; + numbackends DESC, + datname ASC; diff --git a/postgres_blocked_queries.sql b/postgres_blocked_queries.sql index b2062c3..741f5ac 100644 --- a/postgres_blocked_queries.sql +++ b/postgres_blocked_queries.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 18:53:54 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Lists PostgreSQL queries blocked along with the pids of those holding the locks blocking them @@ -20,11 +20,11 @@ -- Tested on PostgreSQL 9.6+, 10.x 11.x, 12.x, 13.0 SELECT - pid, - usename, - pg_blocking_pids(pid) AS blocked_by_pids, - query AS blocked_query + pid, + usename, + pg_blocking_pids(pid) AS blocked_by_pids, + query AS blocked_query FROM - pg_stat_activity + pg_stat_activity WHERE - cardinality(pg_blocking_pids(pid)) > 0; + cardinality(pg_blocking_pids(pid)) > 0; diff --git a/postgres_columns_null.sql b/postgres_columns_null.sql index 2c55746..d531231 100644 --- a/postgres_columns_null.sql +++ b/postgres_columns_null.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2020-10-20 11:01:44 +0100 (Tue, 20 Oct 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- @@ -19,44 +19,44 @@ SELECT - nspname, - relname, - attname, - typname, - (stanullfrac*100)::int AS null_percent + nspname, + relname, + attname, + typname, + (stanullfrac*100)::int AS null_percent FROM - pg_class c + pg_class c JOIN - pg_namespace ns + pg_namespace ns ON - (ns.oid=relnamespace) + (ns.oid=relnamespace) JOIN - pg_attribute + pg_attribute ON - (c.oid=attrelid) + (c.oid=attrelid) JOIN - pg_type t + pg_type t ON - (t.oid=atttypid) + (t.oid=atttypid) JOIN - pg_statistic + pg_statistic ON - (c.oid=starelid AND staattnum=attnum) + (c.oid=starelid AND staattnum=attnum) WHERE - (stanullfrac*100)::int = 100 - AND - nspname NOT LIKE E'pg\\_%' - AND - nspname != 'information_schema' - AND - relkind = 'r' - AND - NOT attisdropped - AND - attstattarget != 0 - -- AND - --reltuples >= 100 + (stanullfrac*100)::int = 100 + AND + nspname NOT LIKE E'pg\\_%' + AND + nspname != 'information_schema' + AND + relkind = 'r' + AND + NOT attisdropped + AND + attstattarget != 0 + -- AND + --reltuples >= 100 ORDER BY - nspname, - relname, - attname; + nspname, + relname, + attname; diff --git a/postgres_columns_useless.sql b/postgres_columns_useless.sql index 0a5d298..a434bfa 100644 --- a/postgres_columns_useless.sql +++ b/postgres_columns_useless.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2020-10-20 11:01:44 +0100 (Tue, 20 Oct 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- @@ -23,56 +23,56 @@ SELECT - nspname, - relname, - attname, - typname, - (stanullfrac*100)::int AS null_percent, - case - when stadistinct >= 0 - then stadistinct - else - abs(stadistinct)*reltuples - end AS "distinct", - case 1 - when stakind1 - then stavalues1 - when stakind2 - then stavalues2 - end AS "values" + nspname, + relname, + attname, + typname, + (stanullfrac*100)::int AS null_percent, + case + when stadistinct >= 0 + then stadistinct + else + abs(stadistinct)*reltuples + end AS "distinct", + case 1 + when stakind1 + then stavalues1 + when stakind2 + then stavalues2 + end AS "values" FROM - pg_class c + pg_class c JOIN - pg_namespace ns + pg_namespace ns ON - (ns.oid=relnamespace) + (ns.oid=relnamespace) JOIN - pg_attribute + pg_attribute ON - (c.oid=attrelid) + (c.oid=attrelid) JOIN - pg_type t + pg_type t ON - (t.oid=atttypid) + (t.oid=atttypid) JOIN - pg_statistic + pg_statistic ON - (c.oid=starelid AND staattnum=attnum) + (c.oid=starelid AND staattnum=attnum) WHERE - nspname NOT LIKE E'pg\\_%' - AND - nspname != 'information_schema' - AND - relkind = 'r' - AND - NOT attisdropped - AND - attstattarget != 0 - AND - reltuples >= 100 - AND - stadistinct BETWEEN 0 AND 1 + nspname NOT LIKE E'pg\\_%' + AND + nspname != 'information_schema' + AND + relkind = 'r' + AND + NOT attisdropped + AND + attstattarget != 0 + AND + reltuples >= 100 + AND + stadistinct BETWEEN 0 AND 1 ORDER BY - nspname, - relname, - attname; + nspname, + relname, + attname; diff --git a/postgres_databases_by_size.sql b/postgres_databases_by_size.sql index 885acc4..cfe8e9d 100644 --- a/postgres_databases_by_size.sql +++ b/postgres_databases_by_size.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 14:54:12 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL databases by size descending @@ -18,9 +18,9 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - datname, - pg_size_pretty(pg_database_size(datname)) + datname, + pg_size_pretty(pg_database_size(datname)) FROM - pg_database + pg_database ORDER - BY pg_database_size(datname) DESC; + BY pg_database_size(datname) DESC; diff --git a/postgres_databases_by_size_if_accessible.sql b/postgres_databases_by_size_if_accessible.sql index 671668e..b17dfb5 100644 --- a/postgres_databases_by_size_if_accessible.sql +++ b/postgres_databases_by_size_if_accessible.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:49:13 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL databases by size descending (only ones you can access) @@ -18,17 +18,17 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - d.datname AS Name, - pg_catalog.pg_get_userbyid(d.datdba) AS Owner, - CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') - THEN pg_catalog.pg_size_pretty(pg_catalog.pg_database_size(d.datname)) - ELSE 'No Access' - END AS SIZE + d.datname AS Name, + pg_catalog.pg_get_userbyid(d.datdba) AS Owner, + CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') + THEN pg_catalog.pg_size_pretty(pg_catalog.pg_database_size(d.datname)) + ELSE 'No Access' + END AS SIZE FROM - pg_catalog.pg_database d + pg_catalog.pg_database d ORDER BY - CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') - THEN pg_catalog.pg_database_size(d.datname) - ELSE NULL - END - DESC; + CASE WHEN pg_catalog.has_database_privilege(d.datname, 'CONNECT') + THEN pg_catalog.pg_database_size(d.datname) + ELSE NULL + END + DESC; diff --git a/postgres_dirs.sql b/postgres_dirs.sql index 71049ae..76caa12 100644 --- a/postgres_dirs.sql +++ b/postgres_dirs.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 10:34:23 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Useful PostgreSQL dir info @@ -20,28 +20,28 @@ -- Tested on PostgreSQL 11.9, 12.x, 13.0 SELECT - current_setting('config_file') AS "config_file", - current_setting('hba_file') AS "hba_file", - current_setting('ident_file') AS "ident_file"; + current_setting('config_file') AS "config_file", + current_setting('hba_file') AS "hba_file", + current_setting('ident_file') AS "ident_file"; SELECT - current_setting('data_directory') AS "data_directory", - current_setting('external_pid_file') AS "external_pid_file"; + current_setting('data_directory') AS "data_directory", + current_setting('external_pid_file') AS "external_pid_file"; SELECT - current_setting('unix_socket_directories') AS "unix_socket_directories", - current_setting('unix_socket_permissions') AS "unix_socket_permissions", - current_setting('unix_socket_group') AS "unix_socket_group"; + current_setting('unix_socket_directories') AS "unix_socket_directories", + current_setting('unix_socket_permissions') AS "unix_socket_permissions", + current_setting('unix_socket_group') AS "unix_socket_group"; SELECT - -- not available on PostgreSQL < 10 - pg_current_logfile(), - current_setting('log_directory') AS "log_directory", -- log - current_setting('log_filename') AS "log_filename"; -- postgresql-%Y-%m-%d_%H%M%S.log + -- not available on PostgreSQL < 10 + pg_current_logfile(), + current_setting('log_directory') AS "log_directory", -- log + current_setting('log_filename') AS "log_filename"; -- postgresql-%Y-%m-%d_%H%M%S.log SELECT - -- CASE WHEN pg_current_logfile() IS NOT NULL THEN pg_ls_logdir() END AS pg_ls_logdir, - pg_ls_waldir(), - -- not available on PostgreSQL <= 11.8 - pg_ls_archive_statusdir(), - pg_ls_tmpdir(); + -- CASE WHEN pg_current_logfile() IS NOT NULL THEN pg_ls_logdir() END AS pg_ls_logdir, + pg_ls_waldir(), + -- not available on PostgreSQL <= 11.8 + pg_ls_archive_statusdir(), + pg_ls_tmpdir(); diff --git a/postgres_dirs_pre10.sql b/postgres_dirs_pre10.sql index b756cb8..8d359a0 100644 --- a/postgres_dirs_pre10.sql +++ b/postgres_dirs_pre10.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 10:34:23 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Useful PostgreSQL dir info @@ -20,31 +20,31 @@ -- Tested on PostgreSQL 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - current_setting('config_file') AS "config_file", - current_setting('hba_file') AS "hba_file", - current_setting('ident_file') AS "ident_file"; + current_setting('config_file') AS "config_file", + current_setting('hba_file') AS "hba_file", + current_setting('ident_file') AS "ident_file"; SELECT - current_setting('data_directory') AS "data_directory", - current_setting('external_pid_file') AS "external_pid_file"; + current_setting('data_directory') AS "data_directory", + current_setting('external_pid_file') AS "external_pid_file"; SELECT -- not available on PostgreSQL < 9.3 - --current_setting('unix_socket_directories') AS "unix_socket_directories", - current_setting('unix_socket_permissions') AS "unix_socket_permissions", - current_setting('unix_socket_group') AS "unix_socket_group"; + --current_setting('unix_socket_directories') AS "unix_socket_directories", + current_setting('unix_socket_permissions') AS "unix_socket_permissions", + current_setting('unix_socket_group') AS "unix_socket_group"; SELECT - -- not available on PostgreSQL < 10 - --pg_current_logfile(), - current_setting('log_directory') AS "log_directory", -- log - current_setting('log_filename') AS "log_filename"; -- postgresql-%Y-%m-%d_%H%M%S.log + -- not available on PostgreSQL < 10 + --pg_current_logfile(), + current_setting('log_directory') AS "log_directory", -- log + current_setting('log_filename') AS "log_filename"; -- postgresql-%Y-%m-%d_%H%M%S.log --SELECT - -- CASE WHEN pg_current_logfile() IS NOT NULL THEN pg_ls_logdir() END AS pg_ls_logdir, - -- not available on PostgreSQL < 10 - --pg_ls_waldir() - -- not available on PostgreSQL <= 11.8 - --pg_ls_archive_statusdir(), - --pg_ls_tmpdir(); + -- CASE WHEN pg_current_logfile() IS NOT NULL THEN pg_ls_logdir() END AS pg_ls_logdir, + -- not available on PostgreSQL < 10 + --pg_ls_waldir() + -- not available on PostgreSQL <= 11.8 + --pg_ls_archive_statusdir(), + --pg_ls_tmpdir(); ; diff --git a/postgres_dirs_pre11.9.sql b/postgres_dirs_pre11.9.sql index fba9e84..7119a91 100644 --- a/postgres_dirs_pre11.9.sql +++ b/postgres_dirs_pre11.9.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 10:34:23 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Useful PostgreSQL dir info @@ -20,29 +20,29 @@ -- Tested on PostgreSQL 10.x, 11.x, 12.x, 13.0 SELECT - current_setting('config_file') AS "config_file", - current_setting('hba_file') AS "hba_file", - current_setting('ident_file') AS "ident_file"; + current_setting('config_file') AS "config_file", + current_setting('hba_file') AS "hba_file", + current_setting('ident_file') AS "ident_file"; SELECT - current_setting('data_directory') AS "data_directory", - current_setting('external_pid_file') AS "external_pid_file"; + current_setting('data_directory') AS "data_directory", + current_setting('external_pid_file') AS "external_pid_file"; SELECT - current_setting('unix_socket_directories') AS "unix_socket_directories", - current_setting('unix_socket_permissions') AS "unix_socket_permissions", - current_setting('unix_socket_group') AS "unix_socket_group"; + current_setting('unix_socket_directories') AS "unix_socket_directories", + current_setting('unix_socket_permissions') AS "unix_socket_permissions", + current_setting('unix_socket_group') AS "unix_socket_group"; SELECT - -- not available on PostgreSQL < 10 - pg_current_logfile(), - current_setting('log_directory') AS "log_directory", -- log - current_setting('log_filename') AS "log_filename"; -- postgresql-%Y-%m-%d_%H%M%S.log + -- not available on PostgreSQL < 10 + pg_current_logfile(), + current_setting('log_directory') AS "log_directory", -- log + current_setting('log_filename') AS "log_filename"; -- postgresql-%Y-%m-%d_%H%M%S.log SELECT - -- CASE WHEN pg_current_logfile() IS NOT NULL THEN pg_ls_logdir() END AS pg_ls_logdir, - pg_ls_waldir() - -- not available on PostgreSQL <= 11.8 - --pg_ls_archive_statusdir(), - --pg_ls_tmpdir(); + -- CASE WHEN pg_current_logfile() IS NOT NULL THEN pg_ls_logdir() END AS pg_ls_logdir, + pg_ls_waldir() + -- not available on PostgreSQL <= 11.8 + --pg_ls_archive_statusdir(), + --pg_ls_tmpdir(); ; diff --git a/postgres_grant_select_all_tables.sql b/postgres_grant_select_all_tables.sql index 6532373..258cb71 100644 --- a/postgres_grant_select_all_tables.sql +++ b/postgres_grant_select_all_tables.sql @@ -2,9 +2,9 @@ -- Author: Hari Sekhon -- Date: 2021-02-01 17:33:17 +0000 (Mon, 01 Feb 2021) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/HariSekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- diff --git a/postgres_idle_sessions.sql b/postgres_idle_sessions.sql index 0e5e92b..66c1975 100644 --- a/postgres_idle_sessions.sql +++ b/postgres_idle_sessions.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 18:34:44 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List PostgreSQL stale connections that have been idle > 10 mins @@ -24,30 +24,30 @@ -- Tested on PostgreSQL 9.2, 10.x, 11.x, 12.x, 13.0 SELECT - rank() over (partition by client_addr order by backend_start ASC) as rank, - pid, - backend_start, - query_start, - state_change, - datname, - usename, - client_addr + rank() over (partition by client_addr order by backend_start ASC) as rank, + pid, + backend_start, + query_start, + state_change, + datname, + usename, + client_addr FROM - pg_stat_activity + pg_stat_activity WHERE - -- don't kill yourself - pid <> pg_backend_pid() - -- AND - -- don't kill your admin tools - --application_name !~ '(?:psql)|(?:pgAdmin.+)' - -- AND - --usename not in ('postgres') - AND - query in ('') - AND - ( - (current_timestamp - query_start) > interval '10 minutes' - OR - (query_start IS NULL AND (current_timestamp - backend_start) > interval '10 minutes') - ) + -- don't kill yourself + pid <> pg_backend_pid() + -- AND + -- don't kill your admin tools + --application_name !~ '(?:psql)|(?:pgAdmin.+)' + -- AND + --usename not in ('postgres') + AND + query in ('') + AND + ( + (current_timestamp - query_start) > interval '10 minutes' + OR + (query_start IS NULL AND (current_timestamp - backend_start) > interval '10 minutes') + ) ; diff --git a/postgres_idle_sessions_current_db_kill.sql b/postgres_idle_sessions_current_db_kill.sql index 790be5e..f7e7687 100644 --- a/postgres_idle_sessions_current_db_kill.sql +++ b/postgres_idle_sessions_current_db_kill.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:00:35 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Kills idle PostgreSQL sessions for the currently select DB that haven't been used in > 15 minutes @@ -20,28 +20,28 @@ -- Tested on PostgreSQL 9.2+, 10.x, 11.x, 12.x, 13.0 SELECT - pg_terminate_backend(pid) + pg_terminate_backend(pid) FROM - pg_stat_activity + pg_stat_activity WHERE - -- don't kill yourself - pid <> pg_backend_pid() - -- AND - -- don't kill your admin tools - --application_name !~ '(?:psql)|(?:pgAdmin.+)' - -- AND - --usename not in ('postgres') - AND - datname = current_database() - AND - query in ('') - AND - state in ('idle', 'idle in transaction', 'idle in transaction (aborted)', 'disabled') - AND - --state_change < current_timestamp - INTERVAL '15' MINUTE; - ( - (current_timestamp - query_start) > interval '15 minutes' - OR - (query_start IS NULL AND (current_timestamp - backend_start) > interval '15 minutes') - ) + -- don't kill yourself + pid <> pg_backend_pid() + -- AND + -- don't kill your admin tools + --application_name !~ '(?:psql)|(?:pgAdmin.+)' + -- AND + --usename not in ('postgres') + AND + datname = current_database() + AND + query in ('') + AND + state in ('idle', 'idle in transaction', 'idle in transaction (aborted)', 'disabled') + AND + --state_change < current_timestamp - INTERVAL '15' MINUTE; + ( + (current_timestamp - query_start) > interval '15 minutes' + OR + (query_start IS NULL AND (current_timestamp - backend_start) > interval '15 minutes') + ) ; diff --git a/postgres_idle_sessions_current_db_kill_pre92.sql b/postgres_idle_sessions_current_db_kill_pre92.sql index df90fa6..3f9586e 100644 --- a/postgres_idle_sessions_current_db_kill_pre92.sql +++ b/postgres_idle_sessions_current_db_kill_pre92.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:00:35 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Kills idle PostgreSQL sessions for the currently select DB that haven't been used in > 15 minutes @@ -20,29 +20,29 @@ -- Tested on PostgreSQL 8.4, 9.0, 9.1 SELECT - pg_terminate_backend(procpid) + pg_terminate_backend(procpid) FROM - pg_stat_activity + pg_stat_activity WHERE - -- don't kill yourself - procpid <> pg_backend_pid() - -- AND - -- don't kill your admin tools - --application_name !~ '(?:psql)|(?:pgAdmin.+)' - -- AND - --usename not in ('postgres') - AND - datname = current_database() - AND - current_query in ('') - AND - waiting - --state in ('idle', 'idle in transaction', 'idle in transaction (aborted)', 'disabled') - AND - --state_change < current_timestamp - INTERVAL '15' MINUTE; - ( - (current_timestamp - query_start) > interval '15 minutes' - OR - (query_start IS NULL AND (current_timestamp - backend_start) > interval '15 minutes') - ) + -- don't kill yourself + procpid <> pg_backend_pid() + -- AND + -- don't kill your admin tools + --application_name !~ '(?:psql)|(?:pgAdmin.+)' + -- AND + --usename not in ('postgres') + AND + datname = current_database() + AND + current_query in ('') + AND + waiting + --state in ('idle', 'idle in transaction', 'idle in transaction (aborted)', 'disabled') + AND + --state_change < current_timestamp - INTERVAL '15' MINUTE; + ( + (current_timestamp - query_start) > interval '15 minutes' + OR + (query_start IS NULL AND (current_timestamp - backend_start) > interval '15 minutes') + ) ; diff --git a/postgres_idle_sessions_kill.sql b/postgres_idle_sessions_kill.sql index 3e25427..b0c92d9 100644 --- a/postgres_idle_sessions_kill.sql +++ b/postgres_idle_sessions_kill.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:00:35 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Kills idle PostgreSQL sessions that haven't been used in > 15 minutes @@ -20,26 +20,26 @@ -- Tested on PostgreSQL 9.2+, 10.x, 11.x, 12.x, 13.0 SELECT - pg_terminate_backend(pid) + pg_terminate_backend(pid) FROM - pg_stat_activity + pg_stat_activity WHERE - -- don't kill yourself - pid <> pg_backend_pid() - -- AND - -- don't kill your admin tools - --application_name !~ '(?:psql)|(?:pgAdmin.+)' - -- AND - --usename not in ('postgres') - AND - query in ('') - AND - state in ('idle', 'idle in transaction', 'idle in transaction (aborted)', 'disabled') - AND - --state_change < current_timestamp - INTERVAL '15' MINUTE; - ( - (current_timestamp - query_start) > interval '15 minutes' - OR - (query_start IS NULL AND (current_timestamp - backend_start) > interval '15 minutes') - ) + -- don't kill yourself + pid <> pg_backend_pid() + -- AND + -- don't kill your admin tools + --application_name !~ '(?:psql)|(?:pgAdmin.+)' + -- AND + --usename not in ('postgres') + AND + query in ('') + AND + state in ('idle', 'idle in transaction', 'idle in transaction (aborted)', 'disabled') + AND + --state_change < current_timestamp - INTERVAL '15' MINUTE; + ( + (current_timestamp - query_start) > interval '15 minutes' + OR + (query_start IS NULL AND (current_timestamp - backend_start) > interval '15 minutes') + ) ; diff --git a/postgres_idle_sessions_pre92.sql b/postgres_idle_sessions_pre92.sql index f2b6293..872dcf3 100644 --- a/postgres_idle_sessions_pre92.sql +++ b/postgres_idle_sessions_pre92.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 18:34:44 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List PostgreSQL stale connections that have been idle > 10 mins @@ -24,30 +24,30 @@ -- Tested on PostgreSQL 8.4, 9.0, 9.1 SELECT - rank() over (partition by client_addr order by backend_start ASC) as rank, - procpid, - backend_start, - query_start, - waiting, - datname, - usename, - client_addr + rank() over (partition by client_addr order by backend_start ASC) as rank, + procpid, + backend_start, + query_start, + waiting, + datname, + usename, + client_addr FROM - pg_stat_activity + pg_stat_activity WHERE - -- don't kill yourself - procpid <> pg_backend_pid() - -- AND - -- don't kill your admin tools - --application_name !~ '(?:psql)|(?:pgAdmin.+)' - -- AND - --usename not in ('postgres') - AND - current_query in ('') - AND - ( - (current_timestamp - query_start) > interval '10 minutes' - OR - (query_start IS NULL AND (current_timestamp - backend_start) > interval '10 minutes') - ) + -- don't kill yourself + procpid <> pg_backend_pid() + -- AND + -- don't kill your admin tools + --application_name !~ '(?:psql)|(?:pgAdmin.+)' + -- AND + --usename not in ('postgres') + AND + current_query in ('') + AND + ( + (current_timestamp - query_start) > interval '10 minutes' + OR + (query_start IS NULL AND (current_timestamp - backend_start) > interval '10 minutes') + ) ; diff --git a/postgres_index_cardinality.sql b/postgres_index_cardinality.sql index 488adb7..9cb0867 100644 --- a/postgres_index_cardinality.sql +++ b/postgres_index_cardinality.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:18:01 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL index cardinality @@ -18,17 +18,17 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - relname, - --relkind, - reltuples AS "cardinality (reltuples)", - relpages + relname, + --relkind, + reltuples AS "cardinality (reltuples)", + relpages FROM - pg_class + pg_class WHERE - relkind = 'i' --- AND --- relname LIKE 'someprefix%'; - AND - relname NOT ILIKE 'pg_%' + relkind = 'i' + -- AND + --relname LIKE 'someprefix%'; + AND + relname NOT ILIKE 'pg_%' ORDER BY - 2 DESC; + 2 DESC; diff --git a/postgres_index_cardinality_with_schema_name.sql b/postgres_index_cardinality_with_schema_name.sql index 8446cb8..b5c3855 100644 --- a/postgres_index_cardinality_with_schema_name.sql +++ b/postgres_index_cardinality_with_schema_name.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:18:01 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL index cardinality with schema name @@ -18,32 +18,32 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x 11.x, 12.x, 13.0 SELECT - schema_name, - relname, - reltuples AS "cardinality (reltuples)", - relpages -FROM ( - SELECT - pg_catalog.pg_namespace.nspname AS schema_name, + schema_name, relname, - reltuples, + reltuples AS "cardinality (reltuples)", relpages - FROM - pg_class - JOIN - pg_namespace ON relnamespace = pg_namespace.oid - WHERE - relkind = 'i' +FROM ( + SELECT + pg_catalog.pg_namespace.nspname AS schema_name, + relname, + reltuples, + relpages + FROM + pg_class + JOIN + pg_namespace ON relnamespace = pg_namespace.oid + WHERE + relkind = 'i' ) t WHERE - schema_name NOT ILIKE 'pg_%' - AND - schema_name <> 'information_schema' + schema_name NOT ILIKE 'pg_%' + AND + schema_name <> 'information_schema' -- AND -- relname LIKE 'someprefix%'; - AND - relname NOT ILIKE 'pg_%' + AND + relname NOT ILIKE 'pg_%' ORDER BY - 3 DESC, - schema_name, - relname; + 3 DESC, + schema_name, + relname; diff --git a/postgres_indexes_cache_hit_ratio.sql b/postgres_indexes_cache_hit_ratio.sql index 40e26bd..edcba15 100644 --- a/postgres_indexes_cache_hit_ratio.sql +++ b/postgres_indexes_cache_hit_ratio.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 15:28:12 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Indexes Cache-hit Ratio @@ -20,10 +20,10 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - SUM(idx_blks_read) AS idx_blks_read, - SUM(idx_blks_hit) AS idx_blks_hit, - SUM(idx_blks_hit) / - GREATEST(SUM(idx_blks_hit) + SUM(idx_blks_read), 1)::float - AS ratio + SUM(idx_blks_read) AS idx_blks_read, + SUM(idx_blks_hit) AS idx_blks_hit, + SUM(idx_blks_hit) / + GREATEST(SUM(idx_blks_hit) + SUM(idx_blks_read), 1)::float + AS ratio FROM - pg_statio_user_indexes; + pg_statio_user_indexes; diff --git a/postgres_indexes_unused.sql b/postgres_indexes_unused.sql index 0360b17..ab0e7db 100644 --- a/postgres_indexes_unused.sql +++ b/postgres_indexes_unused.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 23:48:47 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL unused indexes @@ -18,21 +18,21 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - relname AS table, - indexrelname AS index, - idx_scan, - idx_tup_read, - idx_tup_fetch, - pg_size_pretty(pg_relation_size(indexrelname::regclass)) + relname AS table, + indexrelname AS index, + idx_scan, + idx_tup_read, + idx_tup_fetch, + pg_size_pretty(pg_relation_size(indexrelname::regclass)) FROM - pg_stat_all_indexes + pg_stat_all_indexes WHERE - schemaname = 'public' - AND - idx_scan = 0 - AND - idx_tup_read = 0 - AND - idx_tup_fetch = 0 + schemaname = 'public' + AND + idx_scan = 0 + AND + idx_tup_read = 0 + AND + idx_tup_fetch = 0 ORDER BY - pg_relation_size(indexrelname::regclass) DESC; + pg_relation_size(indexrelname::regclass) DESC; diff --git a/postgres_info.sql b/postgres_info.sql index 1e220f2..34ae61e 100644 --- a/postgres_info.sql +++ b/postgres_info.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 10:34:23 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Useful PostgreSQL functions & info @@ -32,9 +32,9 @@ -- version() returns a long human readable string, hence we split from others SELECTs eg. -- PostgreSQL 12.3 (Debian 12.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit SELECT - version(), - current_setting('server_version') AS "server_version", - current_setting('server_version_num') AS "server_version_num"; + version(), + current_setting('server_version') AS "server_version", + current_setting('server_version_num') AS "server_version_num"; -- ========================================================================== -- -- S e r v e r D e t a i l s @@ -44,18 +44,18 @@ SELECT --\pset title 'PostgreSQL Server Details' SELECT - pg_postmaster_start_time(), - pg_conf_load_time(), - current_setting('logging_collector') AS "logging_collector", - current_setting('log_destination') AS "log_destination", - -- not available in Postgres 9 - -- doesn't work because it still checks if pg_current_logfile() is valid and neither eval or execute seem to work around this - -- CASE WHEN current_setting('server_version_num')::int > 100000 THEN pg_current_logfile() ELSE NULL END as pg_current_logfile - pg_current_logfile() - -- current_setting('log_directory') AS "log_directory", -- log - -- current_setting('log_filename') AS "log_filename", -- postgresql-%Y-%m-%d_%H%M%S.log - -- not available on Postgres 10 - --pg_jit_available() + pg_postmaster_start_time(), + pg_conf_load_time(), + current_setting('logging_collector') AS "logging_collector", + current_setting('log_destination') AS "log_destination", + -- not available in Postgres 9 + -- doesn't work because it still checks if pg_current_logfile() is valid and neither eval or execute seem to work around this + -- CASE WHEN current_setting('server_version_num')::int > 100000 THEN pg_current_logfile() ELSE NULL END as pg_current_logfile + pg_current_logfile() + -- current_setting('log_directory') AS "log_directory", -- log + -- current_setting('log_filename') AS "log_filename", -- postgresql-%Y-%m-%d_%H%M%S.log + -- not available on Postgres 10 + --pg_jit_available() ; -- SELECT pg_reload_conf(), pg_rotate_logfile(); @@ -87,18 +87,18 @@ SELECT --\pset title 'Config Files' SELECT - current_setting('config_file') AS "config_file", - current_setting('hba_file') AS "hba_file", - current_setting('ident_file') AS "ident_file"; + current_setting('config_file') AS "config_file", + current_setting('hba_file') AS "hba_file", + current_setting('ident_file') AS "ident_file"; \echo --\pset title 'PostgreSQL Data Directory & Unix Sockets' SELECT - current_setting('data_directory') AS "data_directory", - current_setting('unix_socket_directories') AS "unix_socket_directories", - current_setting('unix_socket_permissions') AS "unix_socket_permissions", - current_setting('unix_socket_group') AS "unix_socket_group"; + current_setting('data_directory') AS "data_directory", + current_setting('unix_socket_directories') AS "unix_socket_directories", + current_setting('unix_socket_permissions') AS "unix_socket_permissions", + current_setting('unix_socket_group') AS "unix_socket_group"; -- ========================================================================== -- -- B u f f e r s & C o n n e c t i o n s @@ -108,13 +108,13 @@ SELECT --\pset title 'Buffers & Connections' SELECT - current_setting('shared_buffers') AS "shared_buffers", - current_setting('work_mem') AS "work_mem", - current_setting('max_connections') AS "max_connections", - current_setting('max_files_per_process') AS "max_files_per_process", -- should be less than ulimit nofiles to avoid “Too many open files” failures - current_setting('track_activities') AS "track_activities", -- for pg_stat / pg_statio family of system views that are used in many other adjacent scripts - current_setting('track_counts') AS "track_counts", -- needed for the autovacuum daemon - current_setting('password_encryption') AS "password_encryption"; + current_setting('shared_buffers') AS "shared_buffers", + current_setting('work_mem') AS "work_mem", + current_setting('max_connections') AS "max_connections", + current_setting('max_files_per_process') AS "max_files_per_process", -- should be less than ulimit nofiles to avoid “Too many open files” failures + current_setting('track_activities') AS "track_activities", -- for pg_stat / pg_statio family of system views that are used in many other adjacent scripts + current_setting('track_counts') AS "track_counts", -- needed for the autovacuum daemon + current_setting('password_encryption') AS "password_encryption"; -- ========================================================================== -- @@ -126,12 +126,12 @@ SELECT -- in SQL the following have special syntax and should be called without parens: current_catalog, current_role, current_schema, current_user, session_user SELECT - current_user, -- aka user, current_role - this is the effective user for permission checking - session_user, -- connection user before superuser SET SESSION AUTHORIZATION - current_schema, - current_catalog, -- SQL standard, same as current_database() - pg_backend_pid(), - current_query(); + current_user, -- aka user, current_role - this is the effective user for permission checking + session_user, -- connection user before superuser SET SESSION AUTHORIZATION + current_schema, + current_catalog, -- SQL standard, same as current_database() + pg_backend_pid(), + current_query(); \echo --\pset title 'Schema search list' @@ -147,16 +147,16 @@ SELECT current_schemas(true) AS "current_schemas(true) - auto-searched schemas"; --\pset title 'Backup & Recovery' SELECT - pg_is_in_backup(), - pg_is_in_recovery(), - pg_backup_start_time(), - 'see postgres_recovery.sql for more info' AS "info"; - -- use postgres_recovery.sql instead of having a big blank table distracting us here - -- the following recovery control functions can only be executed during recovery - to get just the above use postgres_funcs.sql - --( CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() END) AS "pg_is_wal_replay_paused()", - --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() END) AS "pg_last_wal_receive_lsn()", - --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() END) AS "pg_last_wal_replay_lsn()", - --( CASE WHEN pg_is_in_recovery() THEN pg_last_xact_replay_timestamp() END) AS "pg_last_xact_replay_timestamp()" + pg_is_in_backup(), + pg_is_in_recovery(), + pg_backup_start_time(), + 'see postgres_recovery.sql for more info' AS "info"; + -- use postgres_recovery.sql instead of having a big blank table distracting us here + -- the following recovery control functions can only be executed during recovery - to get just the above use postgres_funcs.sql + --( CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() END) AS "pg_is_wal_replay_paused()", + --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() END) AS "pg_last_wal_receive_lsn()", + --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() END) AS "pg_last_wal_replay_lsn()", + --( CASE WHEN pg_is_in_recovery() THEN pg_last_xact_replay_timestamp() END) AS "pg_last_xact_replay_timestamp()" --SELECT -- pg_ls_logdir(), @@ -173,10 +173,10 @@ SELECT --\pset title 'Networking' SELECT - inet_client_addr(), - inet_client_addr(), - inet_server_addr(), - inet_server_port(); + inet_client_addr(), + inet_client_addr(), + inet_server_addr(), + inet_server_port(); -- causes 0 rows when mixed with other select funcs --SELECT pg_listening_channels(); @@ -193,23 +193,23 @@ SELECT --\pset title 'Date & Time' SELECT - -- current timestamps even inside transactions/functions - now(), - timeofday(), -- human string timestamp with timezone - - -- at start of current transaction for consistency, includes +offset timezone - current_timestamp(2) AS "current_timestamp(2)", -- secs precision of 2 decimal places, includes +offset timezone - current_date, - current_time(1) AS "current_time(1)", -- includes +offset timezone - localtime, - localtimestamp(1) AS "localtimestamp(1)", - - -- provide current timestamps even inside transactions/functions - -- now() - -- timeofday(), -- human string timestamp with timezone - clock_timestamp(), -- current date + time (changes throughout function) - statement_timestamp(), - transaction_timestamp() -- same as CURRENT_TIMESTAMP + -- current timestamps even inside transactions/functions + now(), + timeofday(), -- human string timestamp with timezone + + -- at start of current transaction for consistency, includes +offset timezone + current_timestamp(2) AS "current_timestamp(2)", -- secs precision of 2 decimal places, includes +offset timezone + current_date, + current_time(1) AS "current_time(1)", -- includes +offset timezone + localtime, + localtimestamp(1) AS "localtimestamp(1)", + + -- provide current timestamps even inside transactions/functions + -- now() + -- timeofday(), -- human string timestamp with timezone + clock_timestamp(), -- current date + time (changes throughout function) + statement_timestamp(), + transaction_timestamp() -- same as CURRENT_TIMESTAMP ; --\pset title diff --git a/postgres_info_pre10.sql b/postgres_info_pre10.sql index c279588..d0ff8a9 100644 --- a/postgres_info_pre10.sql +++ b/postgres_info_pre10.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 10:34:23 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Useful PostgreSQL functions & info @@ -30,9 +30,9 @@ -- version() returns a long human readable string, hence we split from others SELECTs eg. -- PostgreSQL 12.3 (Debian 12.3-1.pgdg100+1) on x86_64-pc-linux-gnu, compiled by gcc (Debian 8.3.0-6) 8.3.0, 64-bit SELECT - version(), - current_setting('server_version') AS "server_version", - current_setting('server_version_num') AS "server_version_num"; + version(), + current_setting('server_version') AS "server_version", + current_setting('server_version_num') AS "server_version_num"; -- ========================================================================== -- -- S e r v e r D e t a i l s @@ -42,18 +42,18 @@ SELECT --\pset title 'PostgreSQL Server Details' SELECT - pg_postmaster_start_time(), - pg_conf_load_time(), - current_setting('logging_collector') AS "logging_collector", - current_setting('log_destination') AS "log_destination" - -- not available in Postgres 9 - -- doesn't work because it still checks if pg_current_logfile() is valid and neither eval or execute seem to work around this - -- CASE WHEN current_setting('server_version_num')::int > 100000 THEN pg_current_logfile() ELSE NULL END as pg_current_logfile - --pg_current_logfile() - -- current_setting('log_directory') AS "log_directory", -- log - -- current_setting('log_filename') AS "log_filename", -- postgresql-%Y-%m-%d_%H%M%S.log - -- not available on Postgres 10 - --pg_jit_available() + pg_postmaster_start_time(), + pg_conf_load_time(), + current_setting('logging_collector') AS "logging_collector", + current_setting('log_destination') AS "log_destination" + -- not available in Postgres 9 + -- doesn't work because it still checks if pg_current_logfile() is valid and neither eval or execute seem to work around this + -- CASE WHEN current_setting('server_version_num')::int > 100000 THEN pg_current_logfile() ELSE NULL END as pg_current_logfile + --pg_current_logfile() + -- current_setting('log_directory') AS "log_directory", -- log + -- current_setting('log_filename') AS "log_filename", -- postgresql-%Y-%m-%d_%H%M%S.log + -- not available on Postgres 10 + --pg_jit_available() ; -- SELECT pg_reload_conf(), pg_rotate_logfile(); @@ -85,19 +85,19 @@ SELECT --\pset title 'Config Files' SELECT - current_setting('config_file') AS "config_file", - current_setting('hba_file') AS "hba_file", - current_setting('ident_file') AS "ident_file"; + current_setting('config_file') AS "config_file", + current_setting('hba_file') AS "hba_file", + current_setting('ident_file') AS "ident_file"; \echo --\pset title 'PostgreSQL Data Directory & Unix Sockets' SELECT - current_setting('data_directory') AS "data_directory", - -- not available on PostgreSQL <= 9.2 - --current_setting('unix_socket_directories') AS "unix_socket_directories", - current_setting('unix_socket_permissions') AS "unix_socket_permissions", - current_setting('unix_socket_group') AS "unix_socket_group"; + current_setting('data_directory') AS "data_directory", + -- not available on PostgreSQL <= 9.2 + --current_setting('unix_socket_directories') AS "unix_socket_directories", + current_setting('unix_socket_permissions') AS "unix_socket_permissions", + current_setting('unix_socket_group') AS "unix_socket_group"; -- ========================================================================== -- -- B u f f e r s & C o n n e c t i o n s @@ -107,13 +107,13 @@ SELECT --\pset title 'Buffers & Connections' SELECT - current_setting('shared_buffers') AS "shared_buffers", - current_setting('work_mem') AS "work_mem", - current_setting('max_connections') AS "max_connections", - current_setting('max_files_per_process') AS "max_files_per_process", -- should be less than ulimit nofiles to avoid “Too many open files” failures - current_setting('track_activities') AS "track_activities", -- for pg_stat / pg_statio family of system views that are used in many other adjacent scripts - current_setting('track_counts') AS "track_counts", -- needed for the autovacuum daemon - current_setting('password_encryption') AS "password_encryption"; + current_setting('shared_buffers') AS "shared_buffers", + current_setting('work_mem') AS "work_mem", + current_setting('max_connections') AS "max_connections", + current_setting('max_files_per_process') AS "max_files_per_process", -- should be less than ulimit nofiles to avoid “Too many open files” failures + current_setting('track_activities') AS "track_activities", -- for pg_stat / pg_statio family of system views that are used in many other adjacent scripts + current_setting('track_counts') AS "track_counts", -- needed for the autovacuum daemon + current_setting('password_encryption') AS "password_encryption"; -- ========================================================================== -- @@ -125,12 +125,12 @@ SELECT -- in SQL the following have special syntax and should be called without parens: current_catalog, current_role, current_schema, current_user, session_user SELECT - current_user, -- aka user, current_role - this is the effective user for permission checking - session_user, -- connection user before superuser SET SESSION AUTHORIZATION - current_schema, - current_catalog, -- SQL standard, same as current_database() - pg_backend_pid(), - current_query(); + current_user, -- aka user, current_role - this is the effective user for permission checking + session_user, -- connection user before superuser SET SESSION AUTHORIZATION + current_schema, + current_catalog, -- SQL standard, same as current_database() + pg_backend_pid(), + current_query(); \echo --\pset title 'Schema search list' @@ -143,19 +143,19 @@ SELECT current_schemas(true) AS "current_schemas(true) - auto-searched schemas"; -- ========================================================================== -- --SELECT - -- not available on PostgreSQL <= 9.2 - --pg_is_in_backup(), - -- not available on PostgreSQL 8.4 - --pg_is_in_recovery(), - -- not available on PostgreSQL <= 9.2 - -- pg_backup_start_time(), + -- not available on PostgreSQL <= 9.2 + --pg_is_in_backup(), + -- not available on PostgreSQL 8.4 + --pg_is_in_recovery(), + -- not available on PostgreSQL <= 9.2 + -- pg_backup_start_time(), -- 'see postgres_recovery.sql for more info' AS "info"; - -- use postgres_recovery.sql instead of having a big blank table distracting us here - -- the following recovery control functions can only be executed during recovery - to get just the above use postgres_funcs.sql - --( CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() END) AS "pg_is_wal_replay_paused()", - --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() END) AS "pg_last_wal_receive_lsn()", - --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() END) AS "pg_last_wal_replay_lsn()", - --( CASE WHEN pg_is_in_recovery() THEN pg_last_xact_replay_timestamp() END) AS "pg_last_xact_replay_timestamp()" + -- use postgres_recovery.sql instead of having a big blank table distracting us here + -- the following recovery control functions can only be executed during recovery - to get just the above use postgres_funcs.sql + --( CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() END) AS "pg_is_wal_replay_paused()", + --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() END) AS "pg_last_wal_receive_lsn()", + --( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() END) AS "pg_last_wal_replay_lsn()", + --( CASE WHEN pg_is_in_recovery() THEN pg_last_xact_replay_timestamp() END) AS "pg_last_xact_replay_timestamp()" --SELECT -- pg_ls_logdir(), @@ -175,10 +175,10 @@ SELECT current_schemas(true) AS "current_schemas(true) - auto-searched schemas"; --\pset title 'Networking' SELECT - inet_client_addr(), - inet_client_addr(), - inet_server_addr(), - inet_server_port(); + inet_client_addr(), + inet_client_addr(), + inet_server_addr(), + inet_server_port(); -- causes 0 rows when mixed with other select funcs --SELECT pg_listening_channels(); @@ -195,23 +195,23 @@ SELECT --\pset title 'Date & Time' SELECT - -- current timestamps even inside transactions/functions - now(), - timeofday(), -- human string timestamp with timezone - - -- at start of current transaction for consistency, includes +offset timezone - current_timestamp(2) AS "current_timestamp(2)", -- secs precision of 2 decimal places, includes +offset timezone - current_date, - current_time(1) AS "current_time(1)", -- includes +offset timezone - localtime, - localtimestamp(1) AS "localtimestamp(1)", - - -- provide current timestamps even inside transactions/functions - -- now() - -- timeofday(), -- human string timestamp with timezone - clock_timestamp(), -- current date + time (changes throughout function) - statement_timestamp(), - transaction_timestamp() -- same as CURRENT_TIMESTAMP + -- current timestamps even inside transactions/functions + now(), + timeofday(), -- human string timestamp with timezone + + -- at start of current transaction for consistency, includes +offset timezone + current_timestamp(2) AS "current_timestamp(2)", -- secs precision of 2 decimal places, includes +offset timezone + current_date, + current_time(1) AS "current_time(1)", -- includes +offset timezone + localtime, + localtimestamp(1) AS "localtimestamp(1)", + + -- provide current timestamps even inside transactions/functions + -- now() + -- timeofday(), -- human string timestamp with timezone + clock_timestamp(), -- current date + time (changes throughout function) + statement_timestamp(), + transaction_timestamp() -- same as CURRENT_TIMESTAMP ; --\pset title diff --git a/postgres_last_analyze.sql b/postgres_last_analyze.sql index 87ee51f..6e3b62d 100644 --- a/postgres_last_analyze.sql +++ b/postgres_last_analyze.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:30:04 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Analyze info for tables with rows modified since last analyze @@ -20,17 +20,17 @@ -- Tested on PostgreSQL 9.4+, 10.x, 11.x, 12.x, 13.0 SELECT - schemaname, - relname, - -- not available on PostgreSQL <= 9.3 - n_mod_since_analyze, - last_analyze, - last_autoanalyze, - -- not available on PostgreSQL <= 9.0 - analyze_count, - autoanalyze_count + schemaname, + relname, + -- not available on PostgreSQL <= 9.3 + n_mod_since_analyze, + last_analyze, + last_autoanalyze, + -- not available on PostgreSQL <= 9.0 + analyze_count, + autoanalyze_count FROM pg_stat_user_tables ORDER BY - n_mod_since_analyze DESC, - last_analyze DESC, - last_autoanalyze DESC; + n_mod_since_analyze DESC, + last_analyze DESC, + last_autoanalyze DESC; diff --git a/postgres_last_analyze_pre94.sql b/postgres_last_analyze_pre94.sql index cbf7adb..ca52251 100644 --- a/postgres_last_analyze_pre94.sql +++ b/postgres_last_analyze_pre94.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:30:04 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Analyze info for tables with rows modified since last analyze @@ -18,17 +18,17 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - schemaname, - relname, - -- not available on PostgreSQL <= 9.3 - --n_mod_since_analyze, - last_analyze, - last_autoanalyze - -- not available on PostgreSQL <= 9.0 - --analyze_count, - --autoanalyze_count + schemaname, + relname, + -- not available on PostgreSQL <= 9.3 + --n_mod_since_analyze, + last_analyze, + last_autoanalyze + -- not available on PostgreSQL <= 9.0 + --analyze_count, + --autoanalyze_count FROM pg_stat_user_tables ORDER BY - --n_mod_since_analyze DESC; - last_analyze DESC, - last_autoanalyze DESC; + --n_mod_since_analyze DESC; + last_analyze DESC, + last_autoanalyze DESC; diff --git a/postgres_last_vacuum.sql b/postgres_last_vacuum.sql index a165f7e..457009f 100644 --- a/postgres_last_vacuum.sql +++ b/postgres_last_vacuum.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:30:04 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Vacuum info for tables with deleted rows @@ -20,20 +20,20 @@ -- Tested on PostgreSQL 9.1+, 10.x, 11.x, 12.x, 13.0 SELECT - schemaname, - relname, - n_live_tup, - n_dead_tup, - n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, - last_vacuum, - last_autovacuum, - -- not available on PostgreSQL <= 9.0 - vacuum_count, - autovacuum_count + schemaname, + relname, + n_live_tup, + n_dead_tup, + n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, + last_vacuum, + last_autovacuum, + -- not available on PostgreSQL <= 9.0 + vacuum_count, + autovacuum_count FROM pg_stat_user_tables WHERE - n_dead_tup > 0 + n_dead_tup > 0 ORDER BY - n_dead_tup DESC, - last_vacuum DESC, - last_autovacuum DESC; + n_dead_tup DESC, + last_vacuum DESC, + last_autovacuum DESC; diff --git a/postgres_last_vacuum_analyze.sql b/postgres_last_vacuum_analyze.sql index ededac1..a9fbdc0 100644 --- a/postgres_last_vacuum_analyze.sql +++ b/postgres_last_vacuum_analyze.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:30:04 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Vacuum and Analyze info @@ -20,26 +20,26 @@ -- Tested on PostgreSQL 9.4+, 10.x, 11.x, 12.x, 13.0 SELECT - schemaname, - relname, - n_live_tup, - n_dead_tup, - n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, - n_mod_since_analyze, - last_vacuum, - last_autovacuum, - last_analyze, - last_autoanalyze, - vacuum_count, - autovacuum_count, - analyze_count, - autoanalyze_count + schemaname, + relname, + n_live_tup, + n_dead_tup, + n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, + n_mod_since_analyze, + last_vacuum, + last_autovacuum, + last_analyze, + last_autoanalyze, + vacuum_count, + autovacuum_count, + analyze_count, + autoanalyze_count FROM - pg_stat_user_tables + pg_stat_user_tables ORDER BY - n_dead_tup DESC, - n_mod_since_analyze DESC, - last_vacuum DESC, - last_analyze DESC, - last_autovacuum DESC, - last_autoanalyze DESC; + n_dead_tup DESC, + n_mod_since_analyze DESC, + last_vacuum DESC, + last_analyze DESC, + last_autovacuum DESC, + last_autoanalyze DESC; diff --git a/postgres_last_vacuum_analyze_pre94.sql b/postgres_last_vacuum_analyze_pre94.sql index b4e4e55..2b4276b 100644 --- a/postgres_last_vacuum_analyze_pre94.sql +++ b/postgres_last_vacuum_analyze_pre94.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:30:04 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Vacuum and Analyze info @@ -18,28 +18,28 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - schemaname, - relname, - n_live_tup, - n_dead_tup, - n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, - -- not available on PostgreSQL <= 9.3 - --n_mod_since_analyze, - last_vacuum, - last_autovacuum, - last_analyze, - last_autoanalyze - -- not available on PostgreSQL <= 9.0 - --vacuum_count, - --autovacuum_count, - -- not available on PostgreSQL <= 9.0 - --analyze_count, - --autoanalyze_count + schemaname, + relname, + n_live_tup, + n_dead_tup, + n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, + -- not available on PostgreSQL <= 9.3 + --n_mod_since_analyze, + last_vacuum, + last_autovacuum, + last_analyze, + last_autoanalyze + -- not available on PostgreSQL <= 9.0 + --vacuum_count, + --autovacuum_count, + -- not available on PostgreSQL <= 9.0 + --analyze_count, + --autoanalyze_count FROM - pg_stat_user_tables + pg_stat_user_tables ORDER BY - n_dead_tup DESC, - last_vacuum DESC, - last_analyze DESC, - last_autovacuum DESC, - last_autoanalyze DESC; + n_dead_tup DESC, + last_vacuum DESC, + last_analyze DESC, + last_autovacuum DESC, + last_autoanalyze DESC; diff --git a/postgres_last_vacuum_pre91.sql b/postgres_last_vacuum_pre91.sql index 880cd2d..dad83b8 100644 --- a/postgres_last_vacuum_pre91.sql +++ b/postgres_last_vacuum_pre91.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:30:04 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Vacuum info for tables with deleted rows @@ -18,20 +18,20 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - schemaname, - relname, - n_live_tup, - n_dead_tup, - n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, - last_vacuum, - last_autovacuum - -- not available on PostgreSQL <= 9.0 - --vacuum_count, - --autovacuum_count + schemaname, + relname, + n_live_tup, + n_dead_tup, + n_dead_tup / GREATEST(n_live_tup + n_dead_tup, 1)::float * 100 AS dead_percentage, + last_vacuum, + last_autovacuum + -- not available on PostgreSQL <= 9.0 + --vacuum_count, + --autovacuum_count FROM pg_stat_user_tables WHERE - n_dead_tup > 0 + n_dead_tup > 0 ORDER BY - n_dead_tup DESC, - last_vacuum DESC, - last_autovacuum DESC; + n_dead_tup DESC, + last_vacuum DESC, + last_autovacuum DESC; diff --git a/postgres_locks.sql b/postgres_locks.sql index 858d553..1490290 100644 --- a/postgres_locks.sql +++ b/postgres_locks.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 18:16:55 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List PostgreSQL Locks @@ -20,28 +20,28 @@ -- https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT - t.schemaname, - t.relname, - -- l.database, -- id number is less useful, take schemaname from join instead - l.locktype, - page, - virtualtransaction, - pid, - mode, - granted + t.schemaname, + t.relname, + -- l.database, -- id number is less useful, take schemaname from join instead + l.locktype, + page, + virtualtransaction, + pid, + mode, + granted FROM - pg_locks l, - --pg_stat_user_tables t - pg_stat_all_tables t + pg_locks l, + --pg_stat_user_tables t + pg_stat_all_tables t WHERE - l.relation = t.relid + l.relation = t.relid ORDER BY - relation ASC; + relation ASC; SELECT - relation::regclass AS relation_regclass, - * + relation::regclass AS relation_regclass, + * FROM - pg_locks + pg_locks WHERE - NOT granted; + NOT granted; diff --git a/postgres_locks_blocked.sql b/postgres_locks_blocked.sql index e3ff168..a2078ab 100644 --- a/postgres_locks_blocked.sql +++ b/postgres_locks_blocked.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 18:16:55 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List PostgreSQL locks blocked @@ -22,35 +22,35 @@ -- https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT - blocked_locks.pid AS blocked_pid, - blocked_activity.usename AS blocked_user, - blocking_locks.pid AS blocking_pid, - blocking_activity.usename AS blocking_user, - blocked_activity.query AS blocked_statement, - blocking_activity.query AS current_statement_in_blocking_process + blocked_locks.pid AS blocked_pid, + blocked_activity.usename AS blocked_user, + blocking_locks.pid AS blocking_pid, + blocking_activity.usename AS blocking_user, + blocked_activity.query AS blocked_statement, + blocking_activity.query AS current_statement_in_blocking_process FROM - pg_catalog.pg_locks AS blocked_locks + pg_catalog.pg_locks AS blocked_locks JOIN - pg_catalog.pg_stat_activity AS blocked_activity + pg_catalog.pg_stat_activity AS blocked_activity ON - blocked_activity.pid = blocked_locks.pid + blocked_activity.pid = blocked_locks.pid JOIN - pg_catalog.pg_locks AS blocking_locks + pg_catalog.pg_locks AS blocking_locks ON - blocking_locks.locktype = blocked_locks.locktype - AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database - AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation - AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page - AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple - AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid - AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid - AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid - AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid - AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid - AND blocking_locks.pid != blocked_locks.pid + blocking_locks.locktype = blocked_locks.locktype + AND blocking_locks.database IS NOT DISTINCT FROM blocked_locks.database + AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation + AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page + AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple + AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid + AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid + AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid + AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid + AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid + AND blocking_locks.pid != blocked_locks.pid JOIN - pg_catalog.pg_stat_activity blocking_activity + pg_catalog.pg_stat_activity blocking_activity ON - blocking_activity.pid = blocking_locks.pid + blocking_activity.pid = blocking_locks.pid WHERE - NOT blocked_locks.granted; + NOT blocked_locks.granted; diff --git a/postgres_locks_blocked_application.sql b/postgres_locks_blocked_application.sql index 0b962c7..8beabd7 100644 --- a/postgres_locks_blocked_application.sql +++ b/postgres_locks_blocked_application.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-07 00:43:13 +0100 (Fri, 07 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- List PostgreSQL locks blocked with application name @@ -22,37 +22,37 @@ -- https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT - blocked_locks.pid AS blocked_pid, - blocked_activity.usename AS blocked_user, - blocking_locks.pid AS blocking_pid, - blocking_activity.usename AS blocking_user, - blocked_activity.query AS blocked_statement, - blocking_activity.query AS current_statement_in_blocking_process, - blocked_activity.application_name AS blocked_application, - blocking_activity.application_name AS blocking_application + blocked_locks.pid AS blocked_pid, + blocked_activity.usename AS blocked_user, + blocking_locks.pid AS blocking_pid, + blocking_activity.usename AS blocking_user, + blocked_activity.query AS blocked_statement, + blocking_activity.query AS current_statement_in_blocking_process, + blocked_activity.application_name AS blocked_application, + blocking_activity.application_name AS blocking_application FROM - pg_catalog.pg_locks blocked_locks + pg_catalog.pg_locks blocked_locks JOIN - pg_catalog.pg_stat_activity blocked_activity + pg_catalog.pg_stat_activity blocked_activity ON - blocked_activity.pid = blocked_locks.pid + blocked_activity.pid = blocked_locks.pid JOIN - pg_catalog.pg_locks blocking_locks + pg_catalog.pg_locks blocking_locks ON - blocking_locks.locktype = blocked_locks.locktype - AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE - AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation - AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page - AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple - AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid - AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid - AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid - AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid - AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid - AND blocking_locks.pid != blocked_locks.pid + blocking_locks.locktype = blocked_locks.locktype + AND blocking_locks.DATABASE IS NOT DISTINCT FROM blocked_locks.DATABASE + AND blocking_locks.relation IS NOT DISTINCT FROM blocked_locks.relation + AND blocking_locks.page IS NOT DISTINCT FROM blocked_locks.page + AND blocking_locks.tuple IS NOT DISTINCT FROM blocked_locks.tuple + AND blocking_locks.virtualxid IS NOT DISTINCT FROM blocked_locks.virtualxid + AND blocking_locks.transactionid IS NOT DISTINCT FROM blocked_locks.transactionid + AND blocking_locks.classid IS NOT DISTINCT FROM blocked_locks.classid + AND blocking_locks.objid IS NOT DISTINCT FROM blocked_locks.objid + AND blocking_locks.objsubid IS NOT DISTINCT FROM blocked_locks.objsubid + AND blocking_locks.pid != blocked_locks.pid JOIN - pg_catalog.pg_stat_activity blocking_activity + pg_catalog.pg_stat_activity blocking_activity ON - blocking_activity.pid = blocking_locks.pid + blocking_activity.pid = blocking_locks.pid WHERE - NOT blocked_locks.granted; + NOT blocked_locks.granted; diff --git a/postgres_locks_query_age.sql b/postgres_locks_query_age.sql index 260aae7..4edfffa 100644 --- a/postgres_locks_query_age.sql +++ b/postgres_locks_query_age.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-07 00:47:45 +0100 (Fri, 07 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL locks with query and age @@ -22,21 +22,21 @@ -- https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT - a.datname, - l.relation::regclass, - l.transactionid, - l.mode, - l.GRANTED, - a.usename, - a.query, - a.query_start, - age(now(), a.query_start) AS "age", - a.pid + a.datname, + l.relation::regclass, + l.transactionid, + l.mode, + l.GRANTED, + a.usename, + a.query, + a.query_start, + age(now(), a.query_start) AS "age", + a.pid FROM - pg_stat_activity a + pg_stat_activity a JOIN - pg_locks l + pg_locks l ON - l.pid = a.pid + l.pid = a.pid ORDER BY - a.query_start; + a.query_start; diff --git a/postgres_locks_query_age_pre92.sql b/postgres_locks_query_age_pre92.sql index bba58f4..9c63bcc 100644 --- a/postgres_locks_query_age_pre92.sql +++ b/postgres_locks_query_age_pre92.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-07 00:47:45 +0100 (Fri, 07 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL locks with query and age for PostgreSQL <= 9.1 @@ -22,25 +22,25 @@ -- https://wiki.postgresql.org/wiki/Lock_Monitoring SELECT - a.datname, - c.relname, - l.transactionid, - l.mode, - l.granted, - a.usename, - a.current_query, - a.query_start, - age(now(), a.query_start) as "age", - a.procpid + a.datname, + c.relname, + l.transactionid, + l.mode, + l.granted, + a.usename, + a.current_query, + a.query_start, + age(now(), a.query_start) as "age", + a.procpid FROM - pg_stat_activity a + pg_stat_activity a JOIN - pg_locks l + pg_locks l ON - l.pid = a.procpid + l.pid = a.procpid JOIN - pg_class c + pg_class c ON - c.oid = l.relation + c.oid = l.relation ORDER BY - a.query_start; + a.query_start; diff --git a/postgres_queries_slow.sql b/postgres_queries_slow.sql index ffd756c..0eb9225 100644 --- a/postgres_queries_slow.sql +++ b/postgres_queries_slow.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:24:59 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL queries currently executing that have taken over 30 secs @@ -22,19 +22,19 @@ -- Tested on PostgreSQL 9.6+, 10x, 11.x, 12.x, 13.0 SELECT - now() - query_start as "runtime", - usename, - datname, - -- not available on PostgreSQL < 9.6 - wait_event, - -- not available on PostgreSQL < 9.2 - state, - -- current_query on PostgreSQL < 9.2 - query + now() - query_start as "runtime", + usename, + datname, + -- not available on PostgreSQL < 9.6 + wait_event, + -- not available on PostgreSQL < 9.2 + state, + -- current_query on PostgreSQL < 9.2 + query FROM - pg_stat_activity + pg_stat_activity WHERE - -- can't use 'runtime' here - now() - query_start > '30 seconds'::interval + -- can't use 'runtime' here + now() - query_start > '30 seconds'::interval ORDER BY - runtime DESC; + runtime DESC; diff --git a/postgres_queries_slow_pre92.sql b/postgres_queries_slow_pre92.sql index 8df9ba8..7379725 100644 --- a/postgres_queries_slow_pre92.sql +++ b/postgres_queries_slow_pre92.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:24:59 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL queries currently executing that have taken over 30 secs @@ -22,19 +22,19 @@ -- Tested on PostgreSQL 8.4, 9.0, 9.1 SELECT - now() - query_start as "runtime", - usename, - datname, - -- not available on PostgreSQL < 9.6 - -- wait_event, - waiting, - -- not available on PostgreSQL < 9.2 - --state, - current_query + now() - query_start as "runtime", + usename, + datname, + -- not available on PostgreSQL < 9.6 + -- wait_event, + waiting, + -- not available on PostgreSQL < 9.2 + --state, + current_query FROM - pg_stat_activity + pg_stat_activity WHERE - -- can't use 'runtime' here - now() - query_start > '30 seconds'::interval + -- can't use 'runtime' here + now() - query_start > '30 seconds'::interval ORDER BY - runtime DESC; + runtime DESC; diff --git a/postgres_queries_slow_pre96.sql b/postgres_queries_slow_pre96.sql index f52091b..dad7aa3 100644 --- a/postgres_queries_slow_pre96.sql +++ b/postgres_queries_slow_pre96.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:24:59 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL queries currently executing that have taken over 30 secs @@ -22,19 +22,19 @@ -- Tested on PostgreSQL 9.2, 9.3, 9.4, 9.5 SELECT - now() - query_start as "runtime", - usename, - datname, - -- not available on PostgreSQL < 9.6 - -- wait_event, - waiting, - -- not available on PostgreSQL < 9.2 - state, - query + now() - query_start as "runtime", + usename, + datname, + -- not available on PostgreSQL < 9.6 + -- wait_event, + waiting, + -- not available on PostgreSQL < 9.2 + state, + query FROM - pg_stat_activity + pg_stat_activity WHERE - -- can't use 'runtime' here - now() - query_start > '30 seconds'::interval + -- can't use 'runtime' here + now() - query_start > '30 seconds'::interval ORDER BY - runtime DESC; + runtime DESC; diff --git a/postgres_query_cache_hit_ratio.sql b/postgres_query_cache_hit_ratio.sql index 5877db0..1b2b453 100644 --- a/postgres_query_cache_hit_ratio.sql +++ b/postgres_query_cache_hit_ratio.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 16:22:42 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL queries cache-hit ratio from pg_stat_statements @@ -25,24 +25,24 @@ CREATE EXTENSION IF NOT EXISTS pg_stat_statements; SELECT - calls, - rows, - shared_blks_hit, - shared_blks_read, - -- using greatest() to avoid divide by zero error, by ensuring we divide by at least 1 - shared_blks_hit / - GREATEST(shared_blks_hit + shared_blks_read, 1)::float AS shared_blks_hit_ratio, - -- casting divisor to float to avoid getting integer maths returning zeros instead of fractional ratios - local_blks_hit, - local_blks_read, - local_blks_hit / - GREATEST(local_blks_hit + local_blks_read, 1)::float AS local_blks_hit_ratio, - query + calls, + rows, + shared_blks_hit, + shared_blks_read, + -- using greatest() to avoid divide by zero error, by ensuring we divide by at least 1 + shared_blks_hit / + GREATEST(shared_blks_hit + shared_blks_read, 1)::float AS shared_blks_hit_ratio, + -- casting divisor to float to avoid getting integer maths returning zeros instead of fractional ratios + local_blks_hit, + local_blks_read, + local_blks_hit / + GREATEST(local_blks_hit + local_blks_read, 1)::float AS local_blks_hit_ratio, + query FROM - pg_stat_statements + pg_stat_statements --ORDER BY rows DESC ORDER BY - shared_blks_hit_ratio DESC, - local_blks_hit_ratio DESC, - rows DESC + shared_blks_hit_ratio DESC, + local_blks_hit_ratio DESC, + rows DESC LIMIT 100; diff --git a/postgres_query_times.sql b/postgres_query_times.sql index 5d397f7..4c6f856 100644 --- a/postgres_query_times.sql +++ b/postgres_query_times.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 16:16:21 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL query times from pg_stat_statements @@ -28,20 +28,20 @@ CREATE EXTENSION IF NOT EXISTS pg_stat_statements; SELECT - calls, - rows, - ROUND((total_exec_time::numeric / 1000), 4) AS total_secs, - -- newer versions of PostgreSQL have mean_exec_time field, don't need to calculate - --ROUND((total_exec_time / 1000 / calls)::numeric, 4) AS average_secs, - ROUND(mean_exec_time::numeric / 1000, 4) AS average_secs, - ROUND(min_exec_time::numeric / 1000, 4) AS min_secs, - ROUND(max_exec_time::numeric / 1000, 4) AS max_secs, - ROUND(stddev_exec_time::numeric / 1000, 4) AS stddev_secs, - query + calls, + rows, + ROUND((total_exec_time::numeric / 1000), 4) AS total_secs, + -- newer versions of PostgreSQL have mean_exec_time field, don't need to calculate + --ROUND((total_exec_time / 1000 / calls)::numeric, 4) AS average_secs, + ROUND(mean_exec_time::numeric / 1000, 4) AS average_secs, + ROUND(min_exec_time::numeric / 1000, 4) AS min_secs, + ROUND(max_exec_time::numeric / 1000, 4) AS max_secs, + ROUND(stddev_exec_time::numeric / 1000, 4) AS stddev_secs, + query FROM - pg_stat_statements + pg_stat_statements ORDER BY - average_secs DESC, - calls DESC, - rows DESC + average_secs DESC, + calls DESC, + rows DESC LIMIT 100; diff --git a/postgres_query_times_pre13.sql b/postgres_query_times_pre13.sql index 070a11f..6f49df9 100644 --- a/postgres_query_times_pre13.sql +++ b/postgres_query_times_pre13.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 16:16:21 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL query times from pg_stat_statements @@ -25,20 +25,20 @@ CREATE EXTENSION IF NOT EXISTS pg_stat_statements; SELECT - calls, - rows, - ROUND((total_time::numeric / 1000), 4) AS total_secs, - -- newer versions of PostgreSQL have mean_time field, don't need to calculate - --ROUND((total_time / 1000 / calls)::numeric, 4) AS average_secs, - ROUND(mean_time::numeric / 1000, 4) AS average_secs, - ROUND(min_time::numeric / 1000, 4) AS min_secs, - ROUND(max_time::numeric / 1000, 4) AS max_secs, - ROUND(stddev_time::numeric / 1000, 4) AS stddev_secs, - query + calls, + rows, + ROUND((total_time::numeric / 1000), 4) AS total_secs, + -- newer versions of PostgreSQL have mean_time field, don't need to calculate + --ROUND((total_time / 1000 / calls)::numeric, 4) AS average_secs, + ROUND(mean_time::numeric / 1000, 4) AS average_secs, + ROUND(min_time::numeric / 1000, 4) AS min_secs, + ROUND(max_time::numeric / 1000, 4) AS max_secs, + ROUND(stddev_time::numeric / 1000, 4) AS stddev_secs, + query FROM - pg_stat_statements + pg_stat_statements ORDER BY - average_secs DESC, - calls DESC, - rows DESC + average_secs DESC, + calls DESC, + rows DESC LIMIT 100; diff --git a/postgres_query_times_pre95.sql b/postgres_query_times_pre95.sql index 08eecae..6324efe 100644 --- a/postgres_query_times_pre95.sql +++ b/postgres_query_times_pre95.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 16:16:21 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL query times from pg_stat_statements for PostgreSQL <= 9.4 @@ -29,16 +29,16 @@ CREATE EXTENSION IF NOT EXISTS pg_stat_statements; SELECT - calls, - rows, - ROUND(total_time::numeric / 1000, 4) AS total_secs, - -- newer versions of PostgreSQL have mean_time field, but we have to calculate on PostgreSQL <= 9.4 - ROUND(total_time::numeric / 1000 / GREATEST(calls, 1), 4) AS average_secs, - query + calls, + rows, + ROUND(total_time::numeric / 1000, 4) AS total_secs, + -- newer versions of PostgreSQL have mean_time field, but we have to calculate on PostgreSQL <= 9.4 + ROUND(total_time::numeric / 1000 / GREATEST(calls, 1), 4) AS average_secs, + query FROM - pg_stat_statements + pg_stat_statements ORDER BY - average_secs DESC, - calls DESC, - rows DESC + average_secs DESC, + calls DESC, + rows DESC LIMIT 100; diff --git a/postgres_recovery.sql b/postgres_recovery.sql index 90b611c..369a95d 100644 --- a/postgres_recovery.sql +++ b/postgres_recovery.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 11:36:42 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL recovery info @@ -20,12 +20,12 @@ -- Tested on PostgreSQL 10.x, 11.x, 12.x, 13.0 SELECT - pg_is_in_backup(), - pg_is_in_recovery(), - pg_backup_start_time(), - -- the following recovery control functions can only be executed during recovery - to get just the above use postgres_funcs.sql - ( CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() END) AS "pg_is_wal_replay_paused()", - ( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() END) AS "pg_last_wal_receive_lsn()", - ( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() END) AS "pg_last_wal_replay_lsn()", - ( CASE WHEN pg_is_in_recovery() THEN pg_last_xact_replay_timestamp() END) AS "pg_last_xact_replay_timestamp()" + pg_is_in_backup(), + pg_is_in_recovery(), + pg_backup_start_time(), + -- the following recovery control functions can only be executed during recovery - to get just the above use postgres_funcs.sql + ( CASE WHEN pg_is_in_recovery() THEN pg_is_wal_replay_paused() END) AS "pg_is_wal_replay_paused()", + ( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_receive_lsn() END) AS "pg_last_wal_receive_lsn()", + ( CASE WHEN pg_is_in_recovery() THEN pg_last_wal_replay_lsn() END) AS "pg_last_wal_replay_lsn()", + ( CASE WHEN pg_is_in_recovery() THEN pg_last_xact_replay_timestamp() END) AS "pg_last_xact_replay_timestamp()" ; diff --git a/postgres_running_queries.sql b/postgres_running_queries.sql index 3ff6ee3..9c26cac 100644 --- a/postgres_running_queries.sql +++ b/postgres_running_queries.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 12:54:34 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Running queries for PostgreSQL >= 9.2 @@ -20,16 +20,16 @@ -- Tested on PostgreSQL 9.2+, 10.x, 11.x, 12.x, 13.0 SELECT - pid, - age(clock_timestamp(), query_start), - usename, - application_name, - query + pid, + age(clock_timestamp(), query_start), + usename, + application_name, + query FROM - pg_stat_activity + pg_stat_activity WHERE - state != 'idle' - AND - query NOT ILIKE '%pg_stat_activity%' + state != 'idle' + AND + query NOT ILIKE '%pg_stat_activity%' ORDER BY - query_start DESC; + query_start DESC; diff --git a/postgres_running_queries_pre92.sql b/postgres_running_queries_pre92.sql index 337993f..961af66 100644 --- a/postgres_running_queries_pre92.sql +++ b/postgres_running_queries_pre92.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 12:52:58 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Running queries for PostgreSQL <= 9.1 @@ -20,15 +20,15 @@ -- Tested on PostgreSQL 8.4, 9.0, 9.1 SELECT - procpid, - age(clock_timestamp(), query_start), - usename, - current_query + procpid, + age(clock_timestamp(), query_start), + usename, + current_query FROM - pg_stat_activity + pg_stat_activity WHERE - current_query != '' - AND - current_query NOT ILIKE '%pg_stat_activity%' + current_query != '' + AND + current_query NOT ILIKE '%pg_stat_activity%' ORDER BY - query_start DESC; + query_start DESC; diff --git a/postgres_sequences_restart_all.sql b/postgres_sequences_restart_all.sql index 3d90ad6..07e285a 100644 --- a/postgres_sequences_restart_all.sql +++ b/postgres_sequences_restart_all.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:32:45 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Restarts all PostgreSQL sequences @@ -18,8 +18,8 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - 'ALTER SEQUENCE ' || relname || ' RESTART;' + 'ALTER SEQUENCE ' || relname || ' RESTART;' FROM - pg_class + pg_class WHERE - relkind = 'S'; + relkind = 'S'; diff --git a/postgres_sessions.sql b/postgres_sessions.sql index fde4dea..43f2550 100644 --- a/postgres_sessions.sql +++ b/postgres_sessions.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:23:40 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL count of sessions in each state @@ -20,18 +20,18 @@ -- Tested on PostgreSQL 10.x, 11.x, 12.x, 13.0 SELECT - pid, - usename, - client_addr, - client_hostname, - client_port, - backend_start, - query_start, - state, - -- not available on PostgreSQL < 10 - backend_type + pid, + usename, + client_addr, + client_hostname, + client_port, + backend_start, + query_start, + state, + -- not available on PostgreSQL < 10 + backend_type FROM - pg_stat_activity + pg_stat_activity ORDER BY - -- not available on PostgreSQL < 10 - backend_type; + -- not available on PostgreSQL < 10 + backend_type; diff --git a/postgres_sessions_pre10.sql b/postgres_sessions_pre10.sql index 826abe5..8900e28 100644 --- a/postgres_sessions_pre10.sql +++ b/postgres_sessions_pre10.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:23:40 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL count of sessions in each state @@ -20,13 +20,13 @@ -- Tested on PostgreSQL 9.2 - 9.6, 10.x, 11.x, 12.x, 13.0 SELECT - pid, - usename, - client_addr, - client_hostname, - client_port, - backend_start, - query_start, - state + pid, + usename, + client_addr, + client_hostname, + client_port, + backend_start, + query_start, + state FROM - pg_stat_activity; + pg_stat_activity; diff --git a/postgres_sessions_state_count.sql b/postgres_sessions_state_count.sql index eeb12a3..1951e82 100644 --- a/postgres_sessions_state_count.sql +++ b/postgres_sessions_state_count.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 00:23:40 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL count of sessions in each state @@ -20,11 +20,11 @@ -- Tested on PostgreSQL 9.2+, 10.x, 11.x, 12.x, 13.0 SELECT - count(1), - state + count(1), + state FROM - pg_stat_activity + pg_stat_activity GROUP BY - state + state ORDER BY - 1 DESC; + 1 DESC; diff --git a/postgres_settings.sql b/postgres_settings.sql index a5332a6..07b246a 100644 --- a/postgres_settings.sql +++ b/postgres_settings.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-12 10:36:55 +0100 (Wed, 12 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL running settings and those set in postgres.conf @@ -22,18 +22,18 @@ --\pset title 'pg_settings' \echo pg_settings: SELECT - name, - setting, - unit, - context + name, + setting, + unit, + context FROM - pg_settings; + pg_settings; \echo --\pset title 'pg_file_settings' \echo pg_file_settings: SELECT - * + * FROM - pg_file_settings; + pg_file_settings; diff --git a/postgres_settings_auth.sql b/postgres_settings_auth.sql index 7687070..1b25634 100644 --- a/postgres_settings_auth.sql +++ b/postgres_settings_auth.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL authentication settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-connection.html#RUNTIME-CONFIG-CONNECTION-AUTHENTICATION SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Connections and Authentication / Authentication - category ILIKE '% / Authentication%'; + -- Connections and Authentication / Authentication + category ILIKE '% / Authentication%'; diff --git a/postgres_settings_autovacuum.sql b/postgres_settings_autovacuum.sql index 6bf7b7d..62a337f 100644 --- a/postgres_settings_autovacuum.sql +++ b/postgres_settings_autovacuum.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL auto vacuum settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-autovacuum.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Autovacuum - category ILIKE '%Autovacuum%'; + -- Autovacuum + category ILIKE '%Autovacuum%'; diff --git a/postgres_settings_client_connection.sql b/postgres_settings_client_connection.sql index 0a70e17..a91bb1b 100644 --- a/postgres_settings_client_connection.sql +++ b/postgres_settings_client_connection.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL client connection default settings @@ -24,24 +24,24 @@ -- https://www.postgresql.org/docs/12/runtime-config-client.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Client Connection Defaults / Statement Behavior - -- Client Connection Defaults / Locale and Formatting - -- Client Connection Defaults / Other Defaults - -- Client Connection Defaults / Shared Library Preloading - category ILIKE '%Client Connection Defaults%'; + -- Client Connection Defaults / Statement Behavior + -- Client Connection Defaults / Locale and Formatting + -- Client Connection Defaults / Other Defaults + -- Client Connection Defaults / Shared Library Preloading + category ILIKE '%Client Connection Defaults%'; diff --git a/postgres_settings_compatibility.sql b/postgres_settings_compatibility.sql index a017abe..bd6aea4 100644 --- a/postgres_settings_compatibility.sql +++ b/postgres_settings_compatibility.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL version and platform compatibility settings @@ -24,22 +24,22 @@ -- https://www.postgresql.org/docs/12/runtime-config-compatible.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Version and Platform Compatibility / Previous PostgreSQL Versions - -- Version and Platform Compatibility / Other Platforms and Clients - category ILIKE '%Compatibility%'; + -- Version and Platform Compatibility / Previous PostgreSQL Versions + -- Version and Platform Compatibility / Other Platforms and Clients + category ILIKE '%Compatibility%'; diff --git a/postgres_settings_connections.sql b/postgres_settings_connections.sql index 64423df..7cefeeb 100644 --- a/postgres_settings_connections.sql +++ b/postgres_settings_connections.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL connection settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-connection.html#RUNTIME-CONFIG-CONNECTION-SETTINGS SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Connections and Authentication / Connection Settings - category ILIKE '% / Connection Settings%'; + -- Connections and Authentication / Connection Settings + category ILIKE '% / Connection Settings%'; diff --git a/postgres_settings_developer.sql b/postgres_settings_developer.sql index 62d8b10..9bbe461 100644 --- a/postgres_settings_developer.sql +++ b/postgres_settings_developer.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL developer options settings @@ -22,21 +22,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-developer.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Developer Options - category ILIKE '%Developer%'; + -- Developer Options + category ILIKE '%Developer%'; diff --git a/postgres_settings_error_handling.sql b/postgres_settings_error_handling.sql index 241bc94..4768b88 100644 --- a/postgres_settings_error_handling.sql +++ b/postgres_settings_error_handling.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL error handling settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-error-handling.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Error Handling - category ILIKE '%Error%'; + -- Error Handling + category ILIKE '%Error%'; diff --git a/postgres_settings_files.sql b/postgres_settings_files.sql index 5438d4e..7898468 100644 --- a/postgres_settings_files.sql +++ b/postgres_settings_files.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL file location settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-file-locations.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- File Locations - category ILIKE '%File Locations%'; + -- File Locations + category ILIKE '%File Locations%'; diff --git a/postgres_settings_locking.sql b/postgres_settings_locking.sql index 465c4b4..0141f4f 100644 --- a/postgres_settings_locking.sql +++ b/postgres_settings_locking.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL lock settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-locks.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Lock Management - category ILIKE '%Lock%'; + -- Lock Management + category ILIKE '%Lock%'; diff --git a/postgres_settings_logging.sql b/postgres_settings_logging.sql index 2bfc0e1..a8e87f7 100644 --- a/postgres_settings_logging.sql +++ b/postgres_settings_logging.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL logging settings @@ -24,22 +24,22 @@ -- https://www.postgresql.org/docs/12/runtime-config-logging.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Reporting and Logging / What to Log - -- broader '%Log%' pulls in WAL settings which we don't want here - category ILIKE '%Logging%'; + -- Reporting and Logging / What to Log + -- broader '%Log%' pulls in WAL settings which we don't want here + category ILIKE '%Logging%'; diff --git a/postgres_settings_memory.sql b/postgres_settings_memory.sql index 28bef4a..f2b4852 100644 --- a/postgres_settings_memory.sql +++ b/postgres_settings_memory.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL memory resource settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-resource.html#RUNTIME-CONFIG-RESOURCE-MEMORY SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Resource Usage / Memory - category ILIKE '%Memory%'; + -- Resource Usage / Memory + category ILIKE '%Memory%'; diff --git a/postgres_settings_misc.sql b/postgres_settings_misc.sql index e1a7f9c..47d11f6 100644 --- a/postgres_settings_misc.sql +++ b/postgres_settings_misc.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL misc settings @@ -26,105 +26,105 @@ -- https://www.postgresql.org/docs/12/runtime-config.html SELECT - name, - setting, - category, - vartype, - short_desc, - enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + category, + vartype, + short_desc, + enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- inverse of all adjacent postgres_settings_*.sql scripts to check for any other settings - -- - -- Connections and Authentication / Authentication - category NOT ILIKE '% / Authentication%' - AND - -- Autovacuum - category NOT ILIKE '%Autovacuum%' - AND - -- Client Connection Defaults / Statement Behavior - -- Client Connection Defaults / Locale and Formatting - -- Client Connection Defaults / Other Defaults - -- Client Connection Defaults / Shared Library Preloading - category NOT ILIKE '%Client Connection Defaults%' - AND - -- Version and Platform Compatibility / Previous PostgreSQL Versions - -- Version and Platform Compatibility / Other Platforms and Clients - category NOT ILIKE '%Compatibility%' - AND - -- Connections and Authentication / Connection Settings - category NOT ILIKE '% / Connection Settings%' - AND - -- Developer Options - category NOT ILIKE '%Developer%' - AND - -- Error Handling - category NOT ILIKE '%Error%' - AND - -- File Locations - category NOT ILIKE '%File Locations%' - AND - -- Lock Management - category NOT ILIKE '%Lock%' - AND - -- Reporting and Logging / What to Log - -- broader '%Log%' pulls in WAL settings which we don't want here - category NOT ILIKE '%Logging%' - AND - -- Resource Usage / Memory - category NOT ILIKE '%Memory%' - AND - -- Preset Options - category NOT ILIKE '%Preset Options%' - AND - -- Query Tuning / Planner Method Configuration - -- Query Tuning / Planner Cost Constants - -- Query Tuning / Other Planner Options - -- Query Tuning / Genetic Query Optimizer - -- Statistics / Query and Index Statistics Collector - -- - -- more general to pull in 'track_activities' from category 'Statistics / Query and Index Statistics Collector' - --category NOT ILIKE '%Query Tuning%' - category NOT ILIKE '%Query%' - AND - -- Replication / Standby Servers - -- Replication / Sending Servers - -- Replication / Master Server - -- Replication / Subscribers - -- Replication - category NOT ILIKE '%Replication%' - AND - -- Resource Usage / Memory - -- Resource Usage / Asynchronous Behavior - -- Resource Usage / Cost-Based Vacuum Delay - -- Resource Usage / Background Writer - -- Resource Usage / Disk - -- Resource Usage / Kernel Resources - category NOT ILIKE '%Resource%' - AND - name NOT ILIKE '%ssl%' - AND - -- Connections and Authentication / SSL - category NOT ILIKE '%SSL%' - AND - short_desc NOT ILIKE '%SSL%' - AND - -- Statistics / Query and Index Statistics Collector - -- Statistics / Monitoring - category NOT ILIKE '%Statistics%' - AND - -- Write-Ahead Log / Settings - -- Write-Ahead Log / Recovery Target - -- Write-Ahead Log / Checkpoints - -- Write-Ahead Log / Archive Recovery - -- Write-Ahead Log / Archiving - category NOT ILIKE '%Write-Ahead Log%' + -- inverse of all adjacent postgres_settings_*.sql scripts to check for any other settings + -- + -- Connections and Authentication / Authentication + category NOT ILIKE '% / Authentication%' + AND + -- Autovacuum + category NOT ILIKE '%Autovacuum%' + AND + -- Client Connection Defaults / Statement Behavior + -- Client Connection Defaults / Locale and Formatting + -- Client Connection Defaults / Other Defaults + -- Client Connection Defaults / Shared Library Preloading + category NOT ILIKE '%Client Connection Defaults%' + AND + -- Version and Platform Compatibility / Previous PostgreSQL Versions + -- Version and Platform Compatibility / Other Platforms and Clients + category NOT ILIKE '%Compatibility%' + AND + -- Connections and Authentication / Connection Settings + category NOT ILIKE '% / Connection Settings%' + AND + -- Developer Options + category NOT ILIKE '%Developer%' + AND + -- Error Handling + category NOT ILIKE '%Error%' + AND + -- File Locations + category NOT ILIKE '%File Locations%' + AND + -- Lock Management + category NOT ILIKE '%Lock%' + AND + -- Reporting and Logging / What to Log + -- broader '%Log%' pulls in WAL settings which we don't want here + category NOT ILIKE '%Logging%' + AND + -- Resource Usage / Memory + category NOT ILIKE '%Memory%' + AND + -- Preset Options + category NOT ILIKE '%Preset Options%' + AND + -- Query Tuning / Planner Method Configuration + -- Query Tuning / Planner Cost Constants + -- Query Tuning / Other Planner Options + -- Query Tuning / Genetic Query Optimizer + -- Statistics / Query and Index Statistics Collector + -- + -- more general to pull in 'track_activities' from category 'Statistics / Query and Index Statistics Collector' + --category NOT ILIKE '%Query Tuning%' + category NOT ILIKE '%Query%' + AND + -- Replication / Standby Servers + -- Replication / Sending Servers + -- Replication / Master Server + -- Replication / Subscribers + -- Replication + category NOT ILIKE '%Replication%' + AND + -- Resource Usage / Memory + -- Resource Usage / Asynchronous Behavior + -- Resource Usage / Cost-Based Vacuum Delay + -- Resource Usage / Background Writer + -- Resource Usage / Disk + -- Resource Usage / Kernel Resources + category NOT ILIKE '%Resource%' + AND + name NOT ILIKE '%ssl%' + AND + -- Connections and Authentication / SSL + category NOT ILIKE '%SSL%' + AND + short_desc NOT ILIKE '%SSL%' + AND + -- Statistics / Query and Index Statistics Collector + -- Statistics / Monitoring + category NOT ILIKE '%Statistics%' + AND + -- Write-Ahead Log / Settings + -- Write-Ahead Log / Recovery Target + -- Write-Ahead Log / Checkpoints + -- Write-Ahead Log / Archive Recovery + -- Write-Ahead Log / Archiving + category NOT ILIKE '%Write-Ahead Log%' ; diff --git a/postgres_settings_preset.sql b/postgres_settings_preset.sql index ff38309..85eda5a 100644 --- a/postgres_settings_preset.sql +++ b/postgres_settings_preset.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL preset options settings @@ -24,21 +24,21 @@ -- https://www.postgresql.org/docs/12/runtime-config-preset.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Preset Options - category ILIKE '%Preset Options%'; + -- Preset Options + category ILIKE '%Preset Options%'; diff --git a/postgres_settings_query_planning.sql b/postgres_settings_query_planning.sql index 6c5d760..2b2c75e 100644 --- a/postgres_settings_query_planning.sql +++ b/postgres_settings_query_planning.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL Query Tuning & Query Statistics Collection settings @@ -24,28 +24,28 @@ -- https://www.postgresql.org/docs/12/runtime-config-query.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Query Tuning / Planner Method Configuration - -- Query Tuning / Planner Cost Constants - -- Query Tuning / Other Planner Options - -- Query Tuning / Genetic Query Optimizer - -- Statistics / Query and Index Statistics Collector - -- - -- more general to pull in 'track_activities' from category 'Statistics / Query and Index Statistics Collector' - --category ILIKE '%Query Tuning%'; - category ILIKE '%Query%'; + -- Query Tuning / Planner Method Configuration + -- Query Tuning / Planner Cost Constants + -- Query Tuning / Other Planner Options + -- Query Tuning / Genetic Query Optimizer + -- Statistics / Query and Index Statistics Collector + -- + -- more general to pull in 'track_activities' from category 'Statistics / Query and Index Statistics Collector' + --category ILIKE '%Query Tuning%'; + category ILIKE '%Query%'; diff --git a/postgres_settings_replication.sql b/postgres_settings_replication.sql index 0df5cf6..6099c23 100644 --- a/postgres_settings_replication.sql +++ b/postgres_settings_replication.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL Replication settings @@ -24,25 +24,25 @@ -- https://www.postgresql.org/docs/12/runtime-config-replication.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Replication / Standby Servers - -- Replication / Sending Servers - -- Replication / Master Server - -- Replication / Subscribers - -- Replication - category ILIKE '%Replication%'; + -- Replication / Standby Servers + -- Replication / Sending Servers + -- Replication / Master Server + -- Replication / Subscribers + -- Replication + category ILIKE '%Replication%'; diff --git a/postgres_settings_resources.sql b/postgres_settings_resources.sql index 9030e80..8376429 100644 --- a/postgres_settings_resources.sql +++ b/postgres_settings_resources.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL resource settings @@ -24,26 +24,26 @@ -- https://www.postgresql.org/docs/12/runtime-config-resource.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Resource Usage / Memory - -- Resource Usage / Asynchronous Behavior - -- Resource Usage / Cost-Based Vacuum Delay - -- Resource Usage / Background Writer - -- Resource Usage / Disk - -- Resource Usage / Kernel Resources - category ILIKE '%Resource%'; + -- Resource Usage / Memory + -- Resource Usage / Asynchronous Behavior + -- Resource Usage / Cost-Based Vacuum Delay + -- Resource Usage / Background Writer + -- Resource Usage / Disk + -- Resource Usage / Kernel Resources + category ILIKE '%Resource%'; diff --git a/postgres_settings_ssl.sql b/postgres_settings_ssl.sql index 90f0839..768579e 100644 --- a/postgres_settings_ssl.sql +++ b/postgres_settings_ssl.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL SSL settings @@ -24,22 +24,22 @@ -- https://www.postgresql.org/docs/12/runtime-config-connection.html#RUNTIME-CONFIG-CONNECTION-SSL SELECT - name, - setting, - -- category - vartype, - short_desc, - enumvals, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category + vartype, + short_desc, + enumvals, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - name ILIKE '%ssl%' - OR - -- Connections and Authentication / SSL - category ILIKE '%SSL%' - OR - short_desc ILIKE '%SSL%'; + name ILIKE '%ssl%' + OR + -- Connections and Authentication / SSL + category ILIKE '%SSL%' + OR + short_desc ILIKE '%SSL%'; diff --git a/postgres_settings_statistics.sql b/postgres_settings_statistics.sql index 15db27f..4ad4205 100644 --- a/postgres_settings_statistics.sql +++ b/postgres_settings_statistics.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL statistics collection settings @@ -24,22 +24,22 @@ -- https://www.postgresql.org/docs/12/runtime-config-statistics.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Statistics / Query and Index Statistics Collector - -- Statistics / Monitoring - category ILIKE '%Statistics%'; + -- Statistics / Query and Index Statistics Collector + -- Statistics / Monitoring + category ILIKE '%Statistics%'; diff --git a/postgres_settings_wal.sql b/postgres_settings_wal.sql index 20db7a9..d93a258 100644 --- a/postgres_settings_wal.sql +++ b/postgres_settings_wal.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-06 18:07:25 +0100 (Thu, 06 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- Show PostgreSQL WAL settings @@ -24,25 +24,25 @@ -- https://www.postgresql.org/docs/12/runtime-config-wal.html SELECT - name, - setting, - -- category, - vartype, - short_desc, - -- enumvals, - source, - min_val, - max_val, - boot_val, - reset_val, - -- not available on PostgreSQL < 9.5 - pending_restart + name, + setting, + -- category, + vartype, + short_desc, + -- enumvals, + source, + min_val, + max_val, + boot_val, + reset_val, + -- not available on PostgreSQL < 9.5 + pending_restart FROM - pg_settings + pg_settings WHERE - -- Write-Ahead Log / Settings - -- Write-Ahead Log / Recovery Target - -- Write-Ahead Log / Checkpoints - -- Write-Ahead Log / Archive Recovery - -- Write-Ahead Log / Archiving - category ILIKE '%Write-Ahead Log%'; + -- Write-Ahead Log / Settings + -- Write-Ahead Log / Recovery Target + -- Write-Ahead Log / Checkpoints + -- Write-Ahead Log / Archive Recovery + -- Write-Ahead Log / Archiving + category ILIKE '%Write-Ahead Log%'; diff --git a/postgres_tables_by_size.sql b/postgres_tables_by_size.sql index 9823507..0de65a3 100644 --- a/postgres_tables_by_size.sql +++ b/postgres_tables_by_size.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 17:54:19 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL tables by size, excluding catalog and information schema @@ -18,19 +18,19 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - --nspname || '.' || relname AS relation, - nspname, - relname, - pg_size_pretty(pg_total_relation_size(C.oid)) AS total_size + --nspname || '.' || relname AS relation, + nspname, + relname, + pg_size_pretty(pg_total_relation_size(C.oid)) AS total_size FROM - pg_class C + pg_class C LEFT JOIN - pg_namespace N ON (N.oid = C.relnamespace) + pg_namespace N ON (N.oid = C.relnamespace) WHERE - nspname NOT IN ('pg_catalog', 'information_schema') - AND - C.relkind <> 'i' - AND - nspname !~ '^pg_toast' + nspname NOT IN ('pg_catalog', 'information_schema') + AND + C.relkind <> 'i' + AND + nspname !~ '^pg_toast' ORDER BY - pg_total_relation_size(C.oid) DESC; + pg_total_relation_size(C.oid) DESC; diff --git a/postgres_tables_by_size_2.sql b/postgres_tables_by_size_2.sql new file mode 100644 index 0000000..6768cbd --- /dev/null +++ b/postgres_tables_by_size_2.sql @@ -0,0 +1,30 @@ +-- +-- Author: Hari Sekhon +-- Date: 2022-05-18 18:06:53 +0100 (Wed, 18 May 2022) +-- +-- vim:ts=4:sts=4:sw=4:et:filetype=sql +-- +-- https://github.com/HariSekhon/SQL-scripts +-- +-- License: see accompanying Hari Sekhon LICENSE file +-- +-- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +-- +-- https://www.linkedin.com/in/HariSekhon +-- + +-- PostgreSQL tables by size +-- +-- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 + +SELECT + schemaname AS table_schema, + relname AS TABLE_NAME, + pg_size_pretty(pg_total_relation_size(relid)) AS total_size, + pg_size_pretty(pg_relation_size(relid)) AS data_size, + pg_size_pretty(pg_total_relation_size(relid) - pg_relation_size(relid)) AS external_size +FROM + pg_catalog.pg_statio_user_tables +ORDER BY + pg_total_relation_size(relid) DESC, + pg_relation_size(relid) DESC; diff --git a/postgres_tables_cache_hit_ratio.sql b/postgres_tables_cache_hit_ratio.sql index 28f2569..e4cb54c 100644 --- a/postgres_tables_cache_hit_ratio.sql +++ b/postgres_tables_cache_hit_ratio.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 14:57:37 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL Tables Cache-hit Ratio @@ -22,10 +22,10 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - SUM(heap_blks_read) AS heap_blks_read, - SUM(heap_blks_hit) AS heap_blks_hit, - SUM(heap_blks_hit) / - GREATEST(SUM(heap_blks_hit) + SUM(heap_blks_read), 1)::float - AS ratio + SUM(heap_blks_read) AS heap_blks_read, + SUM(heap_blks_hit) AS heap_blks_hit, + SUM(heap_blks_hit) / + GREATEST(SUM(heap_blks_hit) + SUM(heap_blks_read), 1)::float + AS ratio FROM - pg_statio_user_tables; + pg_statio_user_tables; diff --git a/postgres_tables_index_usage.sql b/postgres_tables_index_usage.sql index 463c816..edd432c 100644 --- a/postgres_tables_index_usage.sql +++ b/postgres_tables_index_usage.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 15:08:23 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL % of times indexes are used on tables by table size descending (might want to tweak to list by % descending) @@ -20,13 +20,13 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - relname AS table, - 100 * idx_scan / GREATEST(seq_scan + idx_scan, 1) AS percent_of_times_index_used, - n_live_tup AS rows_in_table + relname AS table, + 100 * idx_scan / GREATEST(seq_scan + idx_scan, 1) AS percent_of_times_index_used, + n_live_tup AS rows_in_table FROM - pg_stat_user_tables + pg_stat_user_tables WHERE - seq_scan + idx_scan > 0 + seq_scan + idx_scan > 0 ORDER BY - rows_in_table DESC, - percent_of_times_index_used DESC; + rows_in_table DESC, + percent_of_times_index_used DESC; diff --git a/postgres_tables_row_estimates.sql b/postgres_tables_row_estimates.sql index 570e56d..9973773 100644 --- a/postgres_tables_row_estimates.sql +++ b/postgres_tables_row_estimates.sql @@ -2,15 +2,15 @@ -- Author: Hari Sekhon -- Date: 2020-08-05 18:21:15 +0100 (Wed, 05 Aug 2020) -- --- vim:ts=2:sts=2:sw=2:et:filetype=sql +-- vim:ts=4:sts=4:sw=4:et:filetype=sql -- --- https://github.com/harisekhon/sql +-- https://github.com/HariSekhon/SQL-scripts -- -- License: see accompanying Hari Sekhon LICENSE file -- -- If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish -- --- https://www.linkedin.com/in/harisekhon +-- https://www.linkedin.com/in/HariSekhon -- -- PostgreSQL table row count estimates in one place @@ -21,11 +21,11 @@ -- Tested on PostgreSQL 8.4, 9.x, 10.x, 11.x, 12.x, 13.0 SELECT - schemaname, - relname, - n_live_tup + schemaname, + relname, + n_live_tup FROM - --pg_stat_all_tables - pg_stat_user_tables + --pg_stat_all_tables + pg_stat_user_tables ORDER BY - n_live_tup DESC; + n_live_tup DESC; diff --git a/setup/bootstrap.sh b/setup/bootstrap.sh index 1d9ac37..351d902 100755 --- a/setup/bootstrap.sh +++ b/setup/bootstrap.sh @@ -4,30 +4,30 @@ # Author: Hari Sekhon # Date: 2019-10-16 10:33:03 +0100 (Wed, 16 Oct 2019) # -# https://github.com/harisekhon/sql-scripts +# https://github.com/HariSekhon/SQL-scripts # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Alpine / Wget: # -# wget -O- https://raw.githubusercontent.com/HariSekhon/sql-scripts/master/setup/bootstrap.sh | sh +# wget -O- https://raw.githubusercontent.com/HariSekhon/SQL-scripts/master/setup/bootstrap.sh | sh # # Curl: # -# curl https://raw.githubusercontent.com/HariSekhon/sql-scripts/master/setup/bootstrap.sh | sh +# curl https://raw.githubusercontent.com/HariSekhon/SQL-scripts/master/setup/bootstrap.sh | sh set -eu [ -n "${DEBUG:-}" ] && set -x srcdir="$(dirname "$0")" -repo="https://github.com/HariSekhon/sql-scripts" +repo="https://github.com/HariSekhon/SQL-scripts" -directory="sql-scripts" +directory="sql" sudo="" [ "$(whoami)" = "root" ] || sudo=sudo @@ -81,4 +81,6 @@ else fi fi -make +if [ -z "${NO_MAKE:-}" ]; then + make +fi diff --git a/setup/ci_bootstrap.sh b/setup/ci_bootstrap.sh index cb7e1ec..61133f2 100755 --- a/setup/ci_bootstrap.sh +++ b/setup/ci_bootstrap.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2020-06-02 17:43:35 +0100 (Tue, 02 Jun 2020) # -# https://github.com/harisekhon/sql-scripts +# https://github.com/HariSekhon/SQL-scripts # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # # Designed to bootstrap all CI systems with retries to make sure the networking, package lists and package repos works before proceeding diff --git a/sonar-project.properties b/sonar-project.properties new file mode 100644 index 0000000..32a5bb0 --- /dev/null +++ b/sonar-project.properties @@ -0,0 +1,42 @@ +# vim:ts=4:sts=4:sw=4:et +# +# Author: Hari Sekhon +# Date: 2016-07-19 18:31:17 +0100 (Tue, 19 Jul 2016) +# +# https://github.com/HariSekhon/SQL-scripts +# +# License: see accompanying Hari Sekhon LICENSE file +# +# If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish +# +# https://www.linkedin.com/in/HariSekhon +# + +# ============================================================================ # +# S o n a r Q u b e +# ============================================================================ # + +sonar.host.url=https://sonarcloud.io + +# Required metadata +sonar.organization=harisekhon +sonar.projectName=SQL-scripts +sonar.projectKey=HariSekhon_SQL-scripts +sonar.projectVersion=1.0 + +sonar.projectDescription=SQL-scripts + +sonar.links.homepage=https://github.com/HariSekhon/SQL-scripts +sonar.links.scm=https://github.com/HariSekhon/SQL-scripts +sonar.links.issue=https://github.com/HariSekhon/SQL-scripts/issues +sonar.links.ci=https://github.com/HariSekhon/SQL-scripts/actions + +# directories to scan (defaults to sonar-project.properties dir otherwise) +sonar.sources=. + +#sonar.language=py + +sonar.sourceEncoding=UTF-8 + +#sonar.exclusions=**/tests/** +#sonar.exclusions=**/zookeeper-*/**/* diff --git a/test.sh b/test.sh index 4154277..04a8996 100755 --- a/test.sh +++ b/test.sh @@ -4,13 +4,13 @@ # Author: Hari Sekhon # Date: 2020-08-12 12:13:30 +0100 (Wed, 12 Aug 2020) # -# https://github.com/harisekhon/sql +# https://github.com/HariSekhon/SQL-scripts # # License: see accompanying Hari Sekhon LICENSE file # # If you're using my code you're welcome to connect with me on LinkedIn and optionally send me feedback to help steer this or other code I publish # -# https://www.linkedin.com/in/harisekhon +# https://www.linkedin.com/in/HariSekhon # set -euo pipefail