From 8bb3730e7b14ba3caba507e28490f9fc43b1125d Mon Sep 17 00:00:00 2001
From: Leah Rowe <leah@libreboot.org>
Date: Wed, 17 Jul 2024 18:15:52 +0100
Subject: [PATCH] cache downloaded files(module) to cache/file/HASH
lib.sh download() is used by subfile handling in git.sh,
e.g. crossgcc tarballs.
they are not currently cached, but are downloaded directly
in place.
cache them, under cache/file/, saved with the name equal
to the checksum, so: cache/file/CHECKSUM
if the given cached file exists, use it as-is for simple
copy, instead of curl. this avoids re-downloading a lot of
crossgcc tarballs, where different coreboot trees may use
some archives that are the same throughout.
Signed-off-by: Leah Rowe <leah@libreboot.org>
---
.gitignore | 1 +
include/lib.sh | 19 ++++++++++---------
2 files changed, 11 insertions(+), 9 deletions(-)
diff --git a/.gitignore b/.gitignore
index 8d1573f..e3fc79b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
*~
*.o
+/cache/
/cbmk.err.log
/repo/
/docs/
diff --git a/include/lib.sh b/include/lib.sh
index 2fb97c4..791ab4e 100644
--- a/include/lib.sh
+++ b/include/lib.sh
@@ -180,18 +180,19 @@ singletree()
download()
{
- dl_fail="y" # 1 url, 2 url backup, 3 destination, 4 checksum
- vendor_checksum "$4" "$3" 2>/dev/null || dl_fail="n"
+ cached="cache/file/$4"
+ dl_fail="n" # 1 url, 2 url backup, 3 destination, 4 checksum
+ vendor_checksum "$4" "$cached" 2>/dev/null && dl_fail="y"
[ "$dl_fail" = "n" ] && e "$3" f && return 0
- x_ mkdir -p "${3%/*}" && for url in "$1" "$2"; do
+ x_ mkdir -p "${3%/*}" cache/file && for url in "$1" "$2"; do
[ "$dl_fail" = "n" ] && break
[ -z "$url" ] && continue
- x_ rm -f "$3"
- curl --location --retry 3 -A "$_ua" "$url" -o "$3" || \
- wget --tries 3 -U "$_ua" "$url" -O "$3" || continue
- vendor_checksum "$4" "$3" || dl_fail="n"
- done;
- [ "$dl_fail" = "y" ] && $err "$1 $2 $3 $4: not downloaded"; return 0
+ x_ rm -f "$cached"
+ curl --location --retry 3 -A "$_ua" "$url" -o "$cached" || \
+ wget --tries 3 -U "$_ua" "$url" -O "$cached" || continue
+ vendor_checksum "$4" "$cached" || dl_fail="n"
+ done; [ "$dl_fail" = "y" ] && $err "$1 $2 $3 $4: not downloaded"
+ [ "$cached" = "$3" ] || cp "$cached" "$3" || $err "!d cp $cached $3"; :
}
vendor_checksum()
--
GitLab