Skip to content

Commit

Permalink
ci(push): update super-linter action
Browse files Browse the repository at this point in the history
  • Loading branch information
streambinder committed Sep 6, 2024
1 parent 89cbe24 commit 201e122
Show file tree
Hide file tree
Showing 33 changed files with 600 additions and 578 deletions.
2 changes: 1 addition & 1 deletion .github/README.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
# Unciæ <a href="https://github.com/streambinder/unciae"><img align="left" src="https://github.com/streambinder.png?size=96"></a>
# Unciæ <a href="https://github.com/streambinder/unciae"><img alt="documentation" align="left" src="https://github.com/streambinder.png?size=96"></a>

This is the safe place to put every mini-project worth to be kept whilst not deserving its own repository.
18 changes: 2 additions & 16 deletions .github/linters/.golangci.yml
Original file line number Diff line number Diff line change
@@ -1,13 +1,4 @@
---
#########################
#########################
## Golang Linter rules ##
#########################
#########################

# configure golangci-lint
# https://github.com/golangci/golangci-lint/blob/master/.golangci.example.yml

issues:
exclude-rules:
- path: _test\.go
Expand All @@ -16,7 +7,6 @@ issues:
- gosec
- goconst
linters:
fast: true
enable:
- gosec
- unconvert
Expand All @@ -28,15 +18,11 @@ linters:
- revive
linters-settings:
errcheck:
# report about assignment of errors to blank identifier
# default is false: such cases aren't reported by default.
check-blank: true
govet:
# report about shadowed variables
check-shadowing: true
enable:
- shadowing
gocyclo:
# minimal code complexity to report, 30 by default
min-complexity: 15
maligned:
# print struct with more effective memory layout or not, false by default
suggest-new: true
3 changes: 3 additions & 0 deletions .github/linters/.mypy.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
[mypy]
ignore_missing_imports = True
explicit_package_bases = True
11 changes: 11 additions & 0 deletions .github/linters/.python-lint
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
[MASTER]

jobs=0

[MESSAGES CONTROL]

disable=
import-error,
missing-module-docstring,
missing-class-docstring,
missing-function-docstring,
15 changes: 12 additions & 3 deletions .github/workflows/push.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
---
name: push

on:
Expand All @@ -24,18 +25,26 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: docker://github/super-linter:latest
- uses: super-linter/super-linter@v7.1.0
env:
GITHUB_TOKEN: ${{ secrets.GH_ACTIONS_UNCIAE }}
VALIDATE_ALL_CODEBASE: false
VALIDATE_PYTHON_PYINK: false
notify:
needs: [commitlint, super-linter]
if: success() && github.event_name == 'push' && github.ref == 'refs/heads/master'
if: >-
success() &&
github.event_name == 'push' &&
github.ref == 'refs/heads/master'
runs-on: ubuntu-latest
steps:
- uses: peter-evans/repository-dispatch@v3
with:
token: ${{ secrets.GH_ACTIONS_UNCIAE }}
repository: streambinder/clavis
event-type: unciae-sync
client-payload: '{"ref": "${{ github.ref }}", "sha": "${{ github.sha }}"}'
client-payload: >-
{
"ref": "${{ github.ref }}",
"sha": "${{ github.sha }}"
}
1 change: 1 addition & 0 deletions arduino/ask_reliability/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ This project wraps up the source code used to run Radio Frequency (RF) communica
Connect the Arduino boards via USB.
Open up `ask_transmitter.ino` and `ask_receiver.ino` as separated projects on Arduino IDE and load them to each specific board.
Follow the flow by connecting to the USB/Serial ports (`ttyUSB` ports mapping could very likely be different):

```bash
# transmitter
screen -S tx /dev/ttyUSB0; screen -X -S tx quit
Expand Down
1 change: 1 addition & 0 deletions backup/restup/config.sample.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
---
tasks:
- repository: /var/restic/database
password: "REDACTED"
Expand Down
2 changes: 1 addition & 1 deletion backup/restup/hooks/acl_pre.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,4 +5,4 @@ source "$(realpath "$(dirname "$0")")/acl_common.sh"

mkdir -p "${DIR_ACL}"
find "${DIR_ACL}" -type f -delete
getfacl -pR / > "/var/acls/root.acl" 2> /dev/null || echo -n
getfacl -pR / >"/var/acls/root.acl" 2>/dev/null || echo -n
2 changes: 1 addition & 1 deletion backup/restup/hooks/db_common.sh
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#!/bin/bash

# shellcheck disable=SC2034
DIR_DB="/var/databases"
DIR_DB="/var/databases"
10 changes: 5 additions & 5 deletions backup/restup/hooks/db_pre.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,8 @@ source "$(realpath "$(dirname "$0")")/db_common.sh"

mkdir -p "${DIR_DB}"
find "${DIR_DB}" -type f -delete
mysql -sN -u root -p'2dkNKCVT9AgFaV8g' -e "SHOW DATABASES;" | \
grep -Ev "(Database|information_schema|performance_schema)" | \
while read -r db; do
mysqldump -u root -p'2dkNKCVT9AgFaV8g' "${db}" --add-drop-table > "${DIR_DB}/${db}.sql"
done
mysql -sN -u root -p'2dkNKCVT9AgFaV8g' -e "SHOW DATABASES;" |
grep -Ev "(Database|information_schema|performance_schema)" |
while read -r db; do
mysqldump -u root -p'2dkNKCVT9AgFaV8g' "${db}" --add-drop-table >"${DIR_DB}/${db}.sql"
done
2 changes: 1 addition & 1 deletion backup/restup/hooks/fw_common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,4 @@

# shellcheck disable=SC2034
FW_FNAME="/etc/sysconfig/iptables"
IPTABLES_SAVE="/sbin/iptables-save"
IPTABLES_SAVE="/sbin/iptables-save"
2 changes: 1 addition & 1 deletion backup/restup/hooks/fw_pre.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@
# shellcheck disable=SC1090,SC1091
source "$(realpath "$(dirname "$0")")/fw_common.sh"

"${IPTABLES_SAVE}" > "${FW_FNAME}"
"${IPTABLES_SAVE}" >"${FW_FNAME}"
148 changes: 68 additions & 80 deletions backup/restup/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,127 +27,115 @@ def __validate_config(self):
for t in self.tasks:
for mandatory_token in ["repository", "password", "path"]:
if mandatory_token not in t or t[mandatory_token] is None:
raise Exception(
"{} key is mandatory for a task object".format(mandatory_token)
raise RuntimeError(
f"{mandatory_token} key is mandatory for a task object"
)
for recommended_token in ["retention"]:
if recommended_token not in t or t[recommended_token] is None:
print(
"Task for {} has not {} token: it's highly recommended.".format(
t["repository"], recommended_token
),
f"Task for {t['repository']} has not {recommended_token} token.",
file=sys.stderr,
)
path_checks = ["repository", "path"]
path_checks += ["prespawn"] if "prespawn" in t else []
path_checks += ["postspawn"] if "postspawn" in t else []
for path_entry in path_checks:
if not os.path.exists(t[path_entry]):
raise Exception(
"{} path {} does not exist".format(path_entry, t[path_entry])
raise RuntimeError(
f"{path_entry} path {t[path_entry]} does not exist"
)

def __wait(self):
for thread in self.__threads:
thread.join()

def __t_print(self, payload, file=sys.stdout):
try:
self.__mutex.acquire()
with self.__mutex:
if isinstance(payload, (bytes, bytearray)):
payload = payload.decode("utf-8")
print(payload)
finally:
self.__mutex.release()
print(payload, file=file)

def __process(self, task):
if "prespawn" in task:
try:
self.__t_print("Running pre-hook {}...".format(task["prespawn"]))
self.__t_print(f"Running pre-hook {task['prespawn']}...")
subprocess.run(task["prespawn"], shell=True, check=True)
except subprocess.CalledProcessError:
self.__t_print(
"Pre-task for {} exited abnormally. Breaking up backup.".format(
task["repository"]
),
f"Pre-task for {task['repository']} exited abnormally. Breaking up backup.",
file=sys.stderr,
)
return

self.__t_print("Spawning {} directory restic backup".format(task["path"]))
self.__t_print(f"Spawning {task['path']} directory restic backup")
regexes = []
if "regexes" in task:
for regex in task["regexes"]:
regexes += ["--iexclude", regex]
pipe_auth = subprocess.Popen(
["echo", "{}".format(task["password"])], stdout=subprocess.PIPE
)
pipe_restic = subprocess.Popen(
[
"restic",
"-r",
task["repository"],
"backup",
task["path"],
"--exclude-caches",
]
+ regexes,
stdin=pipe_auth.stdout,
stdout=subprocess.PIPE,
)
pipe_auth.stdout.close()
pipe_out, pipe_err = pipe_restic.communicate()
if pipe_err is not None:
self.__t_print(
"Unable to backup {} repository: {}".format(
task["respository"], str(pipe_err)
),
file=sys.stderr,
)
return
self.__t_print(pipe_out)

if "postspawn" in task:
try:
self.__t_print("Running post-hook {}...".format(task["postspawn"]))
subprocess.run(task["postspawn"], shell=True, check=True)
except subprocess.CalledProcessError:
self.__t_print(
"Post-task for {} exited abnormally".format(task["repository"]),
file=sys.stderr,
)

if "retention" in task:
self.__t_print("Enforcing {} retention...".format(task["retention"]))
pipe_auth = subprocess.Popen(
["echo", "{}".format(task["password"])], stdout=subprocess.PIPE
)
pipe_restic = subprocess.Popen(
with subprocess.Popen(
["echo", task["password"]], stdout=subprocess.PIPE
) as pipe_auth:
with subprocess.Popen(
[
"restic",
"-r",
task["repository"],
"forget",
"--prune",
"--keep-within",
task["retention"],
],
"backup",
task["path"],
"--exclude-caches",
]
+ regexes,
stdin=pipe_auth.stdout,
stdout=subprocess.PIPE,
)
pipe_auth.stdout.close()
pipe_out, pipe_err = pipe_restic.communicate()
if pipe_err is not None:
) as pipe_restic:
pipe_out, pipe_err = pipe_restic.communicate()
if pipe_err is not None:
self.__t_print(
f"Unable to backup {task['respository']} repository: {pipe_err}",
file=sys.stderr,
)
return
self.__t_print(pipe_out)

if "postspawn" in task:
try:
self.__t_print(f"Running post-hook {task['postspawn']}...")
subprocess.run(task["postspawn"], shell=True, check=True)
except subprocess.CalledProcessError:
self.__t_print(
"Unable to apply retention on {} repository: {}".format(
task["respository"], str(pipe_err)
),
f"Post-task for {task['repository']} exited abnormally",
file=sys.stderr,
)
return
self.__t_print(pipe_out)

self.__t_print("Repository {} updated".format(task["repository"]))
if "retention" in task:
self.__t_print(f"Enforcing {task['retention']} retention...")
with subprocess.Popen(
["echo", task["password"]], stdout=subprocess.PIPE
) as pipe_auth:
with subprocess.Popen(
[
"restic",
"-r",
task["repository"],
"forget",
"--prune",
"--keep-within",
task["retention"],
],
stdin=pipe_auth.stdout,
stdout=subprocess.PIPE,
) as pipe_restic:
pipe_out, pipe_err = pipe_restic.communicate()
if pipe_err is not None:
self.__t_print(
f"Unable to apply retention on {task['respository']} repository: "
+ pipe_err,
file=sys.stderr,
)
return
self.__t_print(pipe_out)

self.__t_print(f"Repository {task['repository']} updated")

def run(self):
for t in self.tasks:
Expand All @@ -160,12 +148,12 @@ def run(self):
if __name__ == "__main__":
os.chdir(os.path.dirname(__file__))

cfg_path = "config.yml"
CFG_PATH = "config.yml"
if len(sys.argv) > 1:
cfg_path = sys.argv[1]
CFG_PATH = sys.argv[1]

try:
with open(cfg_path, "r") as yml_stream:
with open(CFG_PATH, "r", encoding="utf-8") as yml_stream:
Restup(yaml.safe_load(yml_stream)).run()
except Exception as e:
except OSError as e:
print(e, file=sys.stderr)
Loading

0 comments on commit 201e122

Please sign in to comment.