Rewrite upload_b2 nightly action against B2 SDK

4 files changed, 140 insertions, 109 deletions

diff --git a/ci/create_nightly_json.py b/ci/create_nightly_json.py
deleted file mode 100644
index 3a1fcd1ef..000000000
--- a/ci/create_nightly_json.py
+++ /dev/null
@@ -1,51 +0,0 @@
-import subprocess
-import sys
-import json
-import datetime
-import urllib.parse
-import sys
-
-def main():
-    files_by_date = {}
-    bucket = sys.argv[1]
-
-    files_lines = execute_cli(f"b2 ls --long b2://{bucket}/nightly/").split("\n")
-    for x in files_lines:
-        parts = x.split(" ", 1)
-        if parts[0]:
-            print(f"Parts[0]: {parts[0]}", flush=True)
-            json_str = execute_cli(f"b2 file info b2://{bucket}/{parts[0]}")
-            data = json.loads(json_str)
-            name = remove_prefix(data['fileName'], "nightly/")
-            url = f"https://f001.backblazeb2.com/file/{bucket}/nightly/{urllib.parse.quote_plus(name)}"
-            sha1 = data['contentSha1']
-            size = int(data['size'])
-            ts = int(data['fileInfo']['src_last_modified_millis'])
-            date = datetime.datetime.fromtimestamp(ts/1000).strftime('%Y-%m-%d')
-            
-            if date not in files_by_date.keys():
-                files_by_date[date] = []
-
-            files_by_date[date].append({
-                                            'name': name,
-                                            'url': url,
-                                            'sha1': sha1,
-                                            'sizeInBytes': size,
-                                         })
-
-    now = datetime.datetime.utcnow().isoformat()
-
-    print(json.dumps({
-                        'last_updated' : now,
-                        'files': files_by_date
-                     }, sort_keys=True, indent=4))
-
-def remove_prefix(text, prefix):
-    return text[text.startswith(prefix) and len(prefix):]
-
-def execute_cli(command):
-    sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
-    return sb.stdout.read().decode("utf-8");
-
-if __name__ == '__main__':
-    sys.exit(main())
\ No newline at end of file
diff --git a/ci/delete_old_binaries.py b/ci/delete_old_binaries.py
deleted file mode 100644
index f7d5d02af..000000000
--- a/ci/delete_old_binaries.py
+++ /dev/null
@@ -1,33 +0,0 @@
-import subprocess
-import sys
-import json
-import datetime
-import urllib.parse
-import sys
-
-def main():
-    files_by_date = {}
-    bucket = sys.argv[1]
-    days_to_keep = int(sys.argv[2])
-    print(f"Looking for binaries to delete older than {days_to_keep} days")
-
-    files_lines = execute_cli(f"b2 ls --long --versions b2://{bucket}/nightly/").split("\n")
-    for x in files_lines:
-        parts = [y for y in x.split(' ') if y]
-
-        if parts and parts[0]:
-            date = datetime.datetime.strptime(parts[2], '%Y-%m-%d').replace(hour=0, minute=0, second=0, microsecond=0)
-            now = datetime.datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
-            delta = now - date
-
-            if delta.days > days_to_keep:
-                print(f'Deleting b2://{bucket}/{parts[5]}')
-                execute_cli(f'b2 rm b2://{bucket}/{parts[5]}')
-
-
-def execute_cli(command):
-    sb = subprocess.Popen(command, shell=True, stdout=subprocess.PIPE)
-    return sb.stdout.read().decode("utf-8");
-
-if __name__ == '__main__':
-    sys.exit(main())
\ No newline at end of file
diff --git a/ci/nightly.py b/ci/nightly.py
new file mode 100644
index 000000000..7a23fd2c2
--- /dev/null
+++ b/ci/nightly.py
@@ -0,0 +1,140 @@
+import os
+import sys
+from zipfile import ZipFile
+from b2sdk.v2 import InMemoryAccountInfo, B2Api
+from datetime import datetime
+import json
+
+UPLOAD_FOLDER = "nightly/"
+
+info   = InMemoryAccountInfo()
+b2_api = B2Api(info)
+application_key_id = os.environ['APPID']
+application_key    = os.environ['APPKEY']
+bucket_name        = os.environ['BUCKET']
+days_to_keep       = os.environ['DAYS_TO_KEEP']
+
+def auth() -> bool:
+	try:
+		realm = b2_api.account_info.get_realm()
+		return True # Already authenticated
+	except:
+		pass        # Not yet authenticated
+
+	err = b2_api.authorize_account("production", application_key_id, application_key)
+	return err == None
+
+def get_bucket():
+	if not auth(): sys.exit(1)
+	return b2_api.get_bucket_by_name(bucket_name)
+
+def remove_prefix(text: str, prefix: str) -> str:
+	return text[text.startswith(prefix) and len(prefix):]
+
+def create_and_upload_artifact_zip(platform: str, artifact: str) -> int:
+	now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
+	destination_zip_name = "odin-{}-nightly+{}.zip".format(platform, now.strftime("%Y-%m-%d"))
+
+	source_zip_name = artifact
+	if not artifact.endswith(".zip"):
+		print(f"Creating archive {destination_zip_name} from {artifact} and uploading to {bucket_name}")
+
+		source_zip_name = destination_zip_name
+		with ZipFile(source_zip_name, 'w') as z:
+			for root, directory, filenames in os.walk(artifact):
+				for file in filenames:
+					file_path = os.path.join(root, file)
+					zip_path  = os.path.join("dist", os.path.relpath(file_path, artifact))
+					z.write(file_path, zip_path)
+
+		if not os.path.exists(source_zip_name):
+			print(f"Error: Newly created ZIP archive {source_zip_name} not found.")
+			return 1
+
+	print("Uploading {} to {}".format(source_zip_name, UPLOAD_FOLDER + destination_zip_name))
+	bucket = get_bucket()
+	res = bucket.upload_local_file(
+		source_zip_name,                   # Local file to upload
+		"nightly/" + destination_zip_name, # B2 destination path
+	)
+	return 0
+
+def prune_artifacts():
+	print(f"Looking for binaries to delete older than {days_to_keep} days")
+
+	bucket = get_bucket()
+	for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=False):
+		# Timestamp is in milliseconds
+		date  = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0)
+		now   = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0)
+		delta = now - date
+
+		if delta.days > int(days_to_keep):
+			print("Deleting {}".format(file.file_name))
+			file.delete()
+
+	return 0
+
+def update_nightly_json():
+	print(f"Updating nightly.json with files {days_to_keep} days or newer")
+
+	files_by_date = {}
+
+	bucket = get_bucket()
+
+	for file, _ in bucket.ls(UPLOAD_FOLDER, latest_only=True):
+		# Timestamp is in milliseconds
+		date = datetime.fromtimestamp(file.upload_timestamp / 1_000.0).replace(hour=0, minute=0, second=0, microsecond=0).strftime('%Y-%m-%d')
+		name = remove_prefix(file.file_name, UPLOAD_FOLDER)
+		sha1 = file.content_sha1
+		size = file.size
+		url  = bucket.get_download_url(file.file_name)
+
+		if date not in files_by_date.keys():
+			files_by_date[date] = []
+
+		files_by_date[date].append({
+			'name':        name,
+			'url':         url,
+			'sha1':        sha1,
+			'sizeInBytes': size,
+		})
+
+	now = datetime.utcnow().isoformat()
+
+	nightly = json.dumps({
+		'last_updated' : now,
+		'files': files_by_date
+	}, sort_keys=True, indent=4, ensure_ascii=False).encode('utf-8')
+
+	res = bucket.upload_bytes(
+		nightly,        # JSON bytes
+		"nightly.json", # B2 destination path
+	)
+	return 0
+
+if __name__ == "__main__":
+	if len(sys.argv) == 1:
+		print("Usage: {} <verb> [arguments]".format(sys.argv[0]))
+		print("\tartifact <platform prefix> <artifact path>\n\t\tCreates and uploads a platform artifact zip.")
+		print("\tprune\n\t\tDeletes old artifacts from bucket")
+		print("\tjson\n\t\tUpdate and upload nightly.json")
+		sys.exit(1)
+	else:
+		command = sys.argv[1].lower()
+		if command == "artifact":
+			if len(sys.argv) != 4:
+				print("Usage: {} artifact <platform prefix> <artifact path>".format(sys.argv[0]))
+				print("Error: Expected artifact command to be given platform prefix and artifact path.\n")
+				sys.exit(1)
+
+			res = create_and_upload_artifact_zip(sys.argv[2], sys.argv[3])
+			sys.exit(res)
+
+		elif command == "prune":
+			res = prune_artifacts()
+			sys.exit(res)
+
+		elif command == "json":
+			res = update_nightly_json()
+			sys.exit(res)
\ No newline at end of file
diff --git a/ci/upload_create_nightly.sh b/ci/upload_create_nightly.sh
deleted file mode 100755
index 8404b33ff..000000000
--- a/ci/upload_create_nightly.sh
+++ /dev/null
@@ -1,25 +0,0 @@
-#!/bin/bash
-
-set -e
-
-bucket=$1
-platform=$2
-artifact=$3
-
-now=$(date +'%Y-%m-%d')
-filename="odin-$platform-nightly+$now.zip"
-
-echo "Creating archive $filename from $artifact and uploading to $bucket"
-
-# If this is already zipped up (done before artifact upload to keep permissions in tact), just move it.
-if [ "${artifact: -4}" == ".zip" ]
-then
-	echo "Artifact already a zip"
-	mkdir -p "output"
-	mv "$artifact" "output/$filename"
-else
-	echo "Artifact needs to be zipped"
-	7z a -bd "output/$filename" -r "$artifact"
-fi
-
-b2 file upload "$bucket" "output/$filename" "nightly/$filename"