Repository navigation
Expand file tree
/
Copy pathbackup_functions.py
More file actions
115 lines (98 loc) · 5.2 KB
/
Copy pathbackup_functions.py
File metadata and controls
115 lines (98 loc) · 5.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
"""
Incremental backup of scraped data directories to Google Cloud Storage.
Project rule: everything we scrape gets periodically mirrored to GCS for redundancy — some sources
(notably the USGS NIMS webcam bucket, a rolling ~15-month archive) cannot be re-scraped later, so the
GCS copy is the durable record. The sync is incremental: files whose blob already exists with the same
size are skipped, so repeated runs only pay for new data.
One-time setup: authenticate with ``gcloud auth application-default login`` (or point
``GOOGLE_APPLICATION_CREDENTIALS`` at a service-account key).
Run manually or from cron, e.g. nightly::
python backup_functions.py --dir pilot_data --prefix pilot_data
"""
import argparse
import os
from typing import Dict
from google.cloud import storage
DEFAULT_BUCKET = "flow_hydro_2_data"
DEFAULT_PROJECT = "hydro-earthnet-db"
# All automated writes live under this namespace so they are auditable and safely disposable without
# ever touching (or making anyone wonder about) the rest of the bucket.
BACKUP_ROOT_PREFIX = "claude_data"
def upload_directory_to_gcs(local_dir: str, bucket_name: str = DEFAULT_BUCKET, prefix: str = "",
project: str = DEFAULT_PROJECT, skip_existing: bool = True) -> Dict[str, int]:
"""
Recursively mirrors a local directory to a GCS bucket, skipping already-uploaded files.
:param local_dir: The local directory to back up.
:type local_dir: str
:param bucket_name: The destination bucket, defaults to "flow_hydro_2_data".
:type bucket_name: str, optional
:param prefix: Blob name prefix inside the bucket (e.g. "pilot_data"), defaults to "".
:type prefix: str, optional
:param project: The GCP project of the bucket, defaults to "hydro-earthnet-db".
:type project: str, optional
:param skip_existing: Skip files whose blob already exists with the same byte size,
defaults to True.
:type skip_existing: bool, optional
:return: A summary dict with "uploaded", "skipped" and "bytes_uploaded" counts.
:rtype: Dict[str, int]
"""
client = storage.Client(project=project)
bucket = client.bucket(bucket_name)
uploaded, skipped, bytes_uploaded = 0, 0, 0
for root, _, files in os.walk(local_dir):
for name in files:
local_path = os.path.join(root, name)
relative = os.path.relpath(local_path, local_dir)
blob_name = "/".join(filter(None, [BACKUP_ROOT_PREFIX, prefix,
relative.replace(os.sep, "/")]))
size = os.path.getsize(local_path)
if skip_existing:
existing = bucket.get_blob(blob_name)
if existing is not None and existing.size == size:
skipped += 1
continue
bucket.blob(blob_name).upload_from_filename(local_path)
uploaded += 1
bytes_uploaded += size
return {"uploaded": uploaded, "skipped": skipped, "bytes_uploaded": bytes_uploaded}
def upload_file(local_path: str, blob_prefix: str, bucket_name: str = DEFAULT_BUCKET,
project: str = DEFAULT_PROJECT) -> None:
"""
Uploads a single file into the claude_data/ namespace (e.g. a scrape registry).
:param local_path: The local file to upload.
:type local_path: str
:param blob_prefix: The prefix under claude_data/ to place the file in.
:type blob_prefix: str
:param bucket_name: The destination bucket, defaults to "flow_hydro_2_data".
:type bucket_name: str, optional
:param project: The GCP project of the bucket, defaults to "hydro-earthnet-db".
:type project: str, optional
:return: None
:rtype: None
"""
client = storage.Client(project=project)
blob_name = "/".join(filter(None, [BACKUP_ROOT_PREFIX, blob_prefix,
os.path.basename(local_path)]))
client.bucket(bucket_name).blob(blob_name).upload_from_filename(local_path)
def main() -> None:
"""
CLI entry point for backing up a directory to GCS.
:return: None
:rtype: None
"""
parser = argparse.ArgumentParser(description="Incrementally back up a directory to GCS.")
parser.add_argument("--dir", required=True, help="Local directory to back up")
parser.add_argument("--bucket", default=DEFAULT_BUCKET, help="Destination GCS bucket")
parser.add_argument("--prefix", default=None,
help="Blob prefix in the bucket (default: the directory's basename)")
parser.add_argument("--project", default=DEFAULT_PROJECT, help="GCP project of the bucket")
parser.add_argument("--force", action="store_true", help="Re-upload even if blobs already exist")
args = parser.parse_args()
prefix = args.prefix if args.prefix is not None else os.path.basename(os.path.normpath(args.dir))
summary = upload_directory_to_gcs(args.dir, bucket_name=args.bucket, prefix=prefix,
project=args.project, skip_existing=not args.force)
print("Backup complete: %d uploaded (%.1f MB), %d skipped (already in gs://%s/%s/%s)" %
(summary["uploaded"], summary["bytes_uploaded"] / 1e6, summary["skipped"],
args.bucket, BACKUP_ROOT_PREFIX, prefix))
if __name__ == "__main__":
main()